Merge branch 'crawler' into 'master'

crawl jobs with missing value

See merge request !3

app/models/location.rb

 class Location < ApplicationRecord
-  CITY_VIETNAM_NUMBER = 70
+  CITY_VIETNAM_NUMBER = 70.freeze
+
   has_many :locations_jobs
   has_many :jobs, through: :locations_jobs
 end

lib/tasks/crawler.rake

 require "open-uri"
-task crawl_jobs: :environment do
+
+namespace :crawl do
+  desc "crawl industries and locations"
+  task crawl_industries_locations: :environment do
+    crawl_industries_and_locations
+  end
+
+  desc "crawl jobs" 
+  task crawl_jobs: :environment do
     job_links = get_job_links(1)
     crawl_jobs(job_links)
-end
+  end
 
-task crawl_industries_locations: :environment do
-  crawl_industries_and_locations
+  desc "crawl all"
+  task all: [:crawl_industries_locations, :crawl_jobs]
 end
 
 def get_job_links(page)
@@ -13,7 +21,7 @@ def get_job_links(page)
   page.times do |i|
     document = Nokogiri::HTML(open("https://careerbuilder.vn/viec-lam/tat-ca-viec-lam-trang-#{i}-vi.html"))
     jobs_xml = document.xpath('//div/a[@class="job_link"]/@href')
-    jobs_xml.each { |i| job_links << i.value}
+    jobs_xml.each { |item| job_links << item.value}
   end
   job_links
 end
@@ -25,13 +33,14 @@ def crawl_company(company_link)
     company_name = document.css(".content .name").text
     return nil if company_name.empty?
 
-    company = Company.find_by(name: company_name)
-    return company if company.present?
-    
     puts company_name
     company_address = document.css(".content p")[1].text
     company_description = document.css(".main-about-us").css('.content').text
-    company = Company.create!(name: company_name, address: company_address, description: company_description)
+
+    Company.find_or_create_by(name: company_name) do |c|
+      c.address = company_address
+      c.description = company_description
+    end
   rescue => exception
     puts exception
     return nil
@@ -55,22 +64,41 @@ def crawl_job(job_link)
     job_company = crawl_company(job_company_link)
     return if job_company.nil?
 
+    ## parse data
     job_detail = document.css('.job-detail-content div.detail-box')
+
     job_location_name = job_detail[0].css('p a').map{ |val| val.text.strip }
     job_locations = Location.where(city: job_location_name)
+
     job_industry_names = job_detail[1].css('ul li')[1].css('p a').map{ |val| val.text.strip }
     job_industries = Industry.where(name: job_industry_names)
-    job_salary = job_detail[2].css('ul li')[0].css('p').text.strip
-    job_experience = job_detail[2].css('ul li')[1].css('p').text.strip
-    job_level = job_detail[2].css('ul li')[2].css('p').text.strip
-    job_expiration_date = job_detail[2].css('ul li')[3].css('p').text.strip
+
+    job_created_at = job_detail[1].css('ul li')[0].css('p').text.strip
+
+    job_salary = ""
+    job_level = ""
+    job_experience = ""
+    job_expiration_date = ""
+    
+    info_detail = job_detail[2].css('ul li')
+    info_detail.count.times do |i|
+      data = info_detail[i].css('p').text.strip
+
+      job_salary = data if info_detail[i].css('.fa.fa-usd').present?
+      job_experience = data if info_detail[i].css('.fa.fa-briefcase').present?
+      job_level = data if info_detail[i].css('.mdi.mdi-account').present?
+      job_expiration_date = data if info_detail[i].css('.mdi.mdi-calendar-check').present?
+    end
+
     job_description = document.css('.job-detail-content .detail-row').to_s
+    ##
 
     Job.find_or_create_by(title: job_title, company_id: job_company.id) do |job|
       job.salary = job_salary
       job.experience = job_experience
       job.level = job_level
       job.expiration_date = job_expiration_date
+      job.created_at = job_created_at
       job.description = job_description
       job.industries << job_industries
       job.locations << job_locations