fix description, requirement from array to string, crawler first page on CareerBuilder

lib/tasks/web_crawler.rake

-# Web crawler
 require 'open-uri'
 require 'open-uri'
 require 'csv'
 require 'csv'
 require 'zip'
 require 'zip'
@@ -6,90 +5,86 @@ require 'zip'
 namespace :job do
 namespace :job do
   desc 'importjob'
   desc 'importjob'
 
 
-  task web_job_import: :environment do
-    url = 'https://careerbuilder.vn/vi/tim-viec-lam/nhan-vien-thiet-ke-thoi-trang.35B6D3AD.html'
-    unparsed_page = HTTParty.get(url)
-    parsed_page ||= Nokogiri::HTML(unparsed_page.body)
-    job_desc = parsed_page.css('div.job-desc')
-    job_detail = parsed_page.css('section.job-detail-content')
-    # title
-    title = job_desc.css('h1.title').text
-    company = job_desc.css('a.job-company-name').text
-    # info box
-    info_box = job_detail.css('div.detail-box')
-    info_box_item = info_box.css('ul li')
-    update_at, industry, type, salary, experience, level, expiration_date = ''
-    city = info_box.first.text.squish.remove('Địa điểm').strip
-    (0..info_box_item.count - 1).each do |part|
-      info = info_box_item[part].text
-      if info.include?(key = 'Ngày cập nhật')
-        update_at = info.squish.remove(key).strip
-      elsif info.include?(key = 'Ngành nghề')
-        industry = info.squish.remove(key).strip
-      elsif info.include?(key = 'Hình thức')
-        type = info.squish.remove(key).strip
-      elsif info.include?(key = 'Lương')
-        salary = info.squish.remove(key).strip
-      elsif info.include?(key = 'Kinh nghiệm')
-        experience = info.squish.remove(key).strip
-      elsif info.include?(key = 'Cấp bậc')
-        level = info.squish.remove(key).strip
-      elsif info.include?(key = 'Hết hạn nộp')
-        expiration_date = info.squish.remove(key).strip
-      end
-    end
-    # benefit
-    job_detail_row = job_detail.css('div.detail-row')
-    benefit_list = []
-    description_list = []
-    requirement_list = []
-    other_info_list = []
-    benefits = job_detail.css('ul.welfare-list li')
-    (0..benefits.count - 1).each do |part|
-      benefit = benefits[part].text.strip
-      benefit_list << benefit
-    end
-    # description - requirment
-    (0..job_detail_row.count - 1).each do |part|
-      job_detail_text = job_detail_row[part].text
-      if job_detail_text.include?('Mô tả Công việc')
-        descriptions = job_detail_row.css('p')
-        (0..descriptions.count - 1).each do |desc|
-          description = descriptions[desc].text.strip
-          description_list << description
+  task web_job_crawler: :environment do
+    parsed_page ||= Nokogiri::HTML(HTTParty.get('https://careerbuilder.vn/viec-lam/tat-ca-viec-lam-vi.html').body)
+    job_item = parsed_page.css('div.job-item')
+    (0..job_item.count - 1).each do |item|
+      job_link = job_item[item].css('div.title a').attribute('href').text
+      unparsed_job_link = HTTParty.get(job_link)
+      parsed_job_link ||= Nokogiri::HTML(unparsed_job_link.body)
+      job_desc = parsed_job_link.css('div.job-desc')
+      job_detail = parsed_job_link.css('section.job-detail-content')
+      # title - company
+      title = job_desc.css('h1.title').text
+      company = job_desc.css('a.job-company-name').text
+      # info box
+      info_box = job_detail.css('div.detail-box')
+      info_box_item = info_box.css('ul li')
+      city_box = info_box.css('div.map a')
+      # city, update_at, industry, type, salary, experience, level, expiration_date
+      update_at, industry, type, salary, experience, level, expiration_date = ''
+      city = city_box.text
+      (0..info_box_item.count - 1).each do |part|
+        info = info_box_item[part].text
+        if info.include?(key = 'Ngày cập nhật')
+          update_at = info.squish.remove(key).strip
+        elsif info.include?(key = 'Ngành nghề')
+          industry = info.squish.remove(key).strip
+        elsif info.include?(key = 'Hình thức')
+          type = info.squish.remove(key).strip
+        elsif info.include?(key = 'Lương')
+          salary = info.squish.remove(key).strip
+        elsif info.include?(key = 'Kinh nghiệm')
+          experience = info.squish.remove(key).strip
+        elsif info.include?(key = 'Cấp bậc')
+          level = info.squish.remove(key).strip
+        elsif info.include?(key = 'Hết hạn nộp')
+          expiration_date = info.squish.remove(key).strip
         end
         end
-      elsif job_detail_text.include?('Yêu Cầu Công Việc')
-        requirements = job_detail_row.css('p')
-        (0..requirements.count - 1).each do |req|
-          requirement = requirements[req].text.strip
-          requirement_list << requirement
+      end
+      # benefit
+      job_detail_row = job_detail.css('div.detail-row')
+      benefit_list = []
+      other_info_list = []
+      benefits = job_detail.css('ul.welfare-list li')
+      (0..benefits.count - 1).each do |part|
+        benefit = benefits[part].text.strip
+        benefit_list << benefit
+      end
+      # description, requirement
+      description, requirement = ''
+      (0..job_detail_row.count - 1).each do |part|
+        job_detail_text = job_detail_row[part].text
+        if job_detail_text.include?('Mô tả Công việc')
+          description = job_detail_text.partition('Mô tả Công việc').last.squish.strip
+        elsif job_detail_text.include?('Yêu Cầu Công Việc')
+          requirement = job_detail_text.partition('Yêu Cầu Công Việc').last.squish.strip
         end
         end
       end
       end
+      # benefit
+      other_info = job_detail.css('div.content_fck ul li')
+      (0..other_info.count - 1).each do |part|
+        info = other_info[part].text.squish.strip
+        other_info_list << info
+      end
+      job = {
+        title: title,
+        company: company,
+        city: city,
+        update_at: update_at,
+        industry: industry,
+        type: type,
+        salary: salary,
+        experences: experience,
+        level: level,
+        # position: position,
+        expiration_date: expiration_date,
+        benefit: benefit_list,
+        description: description,
+        requirement: requirement,
+        other_info: other_info_list
+      }
+      puts job
     end
     end
-
-    # other info
-    other_info = job_detail.css('div.content_fck ul li')
-    (0..other_info.count - 1).each do |part|
-      info = other_info[part].text.squish.strip
-      other_info_list << info
-    end
-
-    job = {
-      title: title,
-      company: company,
-      city: city,
-      update_at: update_at,
-      industry: industry,
-      type: type,
-      salary: salary,
-      experences: experience,
-      level: level,
-      # position: position,
-      expiration_date: expiration_date,
-      benefit: benefit_list,
-      description: description_list,
-      requirement: requirement_list,
-      other_info: other_info_list
-    }
   end
   end
 end
 end