fix description, requirement from array to string, crawler first page on CareerBuilder

lib/tasks/web_crawler.rake

-# Web crawler
 require 'open-uri'
 require 'open-uri'
 require 'csv'
 require 'csv'
 require 'zip'
 require 'zip'
@@ -6,20 +5,25 @@ require 'zip'
 namespace :job do
 namespace :job do
   desc 'importjob'
   desc 'importjob'
 
 
-  task web_job_import: :environment do
-    url = 'https://careerbuilder.vn/vi/tim-viec-lam/nhan-vien-thiet-ke-thoi-trang.35B6D3AD.html'
-    unparsed_page = HTTParty.get(url)
-    parsed_page ||= Nokogiri::HTML(unparsed_page.body)
-    job_desc = parsed_page.css('div.job-desc')
-    job_detail = parsed_page.css('section.job-detail-content')
-    # title
+  task web_job_crawler: :environment do
+    parsed_page ||= Nokogiri::HTML(HTTParty.get('https://careerbuilder.vn/viec-lam/tat-ca-viec-lam-vi.html').body)
+    job_item = parsed_page.css('div.job-item')
+    (0..job_item.count - 1).each do |item|
+      job_link = job_item[item].css('div.title a').attribute('href').text
+      unparsed_job_link = HTTParty.get(job_link)
+      parsed_job_link ||= Nokogiri::HTML(unparsed_job_link.body)
+      job_desc = parsed_job_link.css('div.job-desc')
+      job_detail = parsed_job_link.css('section.job-detail-content')
+      # title - company
       title = job_desc.css('h1.title').text
       title = job_desc.css('h1.title').text
       company = job_desc.css('a.job-company-name').text
       company = job_desc.css('a.job-company-name').text
       # info box
       # info box
       info_box = job_detail.css('div.detail-box')
       info_box = job_detail.css('div.detail-box')
       info_box_item = info_box.css('ul li')
       info_box_item = info_box.css('ul li')
+      city_box = info_box.css('div.map a')
+      # city, update_at, industry, type, salary, experience, level, expiration_date
       update_at, industry, type, salary, experience, level, expiration_date = ''
       update_at, industry, type, salary, experience, level, expiration_date = ''
-    city = info_box.first.text.squish.remove('Địa điểm').strip
+      city = city_box.text
       (0..info_box_item.count - 1).each do |part|
       (0..info_box_item.count - 1).each do |part|
         info = info_box_item[part].text
         info = info_box_item[part].text
         if info.include?(key = 'Ngày cập nhật')
         if info.include?(key = 'Ngày cập nhật')
@@ -41,39 +45,28 @@ namespace :job do
       # benefit
       # benefit
       job_detail_row = job_detail.css('div.detail-row')
       job_detail_row = job_detail.css('div.detail-row')
       benefit_list = []
       benefit_list = []
-    description_list = []
-    requirement_list = []
       other_info_list = []
       other_info_list = []
       benefits = job_detail.css('ul.welfare-list li')
       benefits = job_detail.css('ul.welfare-list li')
       (0..benefits.count - 1).each do |part|
       (0..benefits.count - 1).each do |part|
         benefit = benefits[part].text.strip
         benefit = benefits[part].text.strip
         benefit_list << benefit
         benefit_list << benefit
       end
       end
-    # description - requirment
+      # description, requirement
+      description, requirement = ''
       (0..job_detail_row.count - 1).each do |part|
       (0..job_detail_row.count - 1).each do |part|
         job_detail_text = job_detail_row[part].text
         job_detail_text = job_detail_row[part].text
         if job_detail_text.include?('Mô tả Công việc')
         if job_detail_text.include?('Mô tả Công việc')
-        descriptions = job_detail_row.css('p')
-        (0..descriptions.count - 1).each do |desc|
-          description = descriptions[desc].text.strip
-          description_list << description
-        end
+          description = job_detail_text.partition('Mô tả Công việc').last.squish.strip
         elsif job_detail_text.include?('Yêu Cầu Công Việc')
         elsif job_detail_text.include?('Yêu Cầu Công Việc')
-        requirements = job_detail_row.css('p')
-        (0..requirements.count - 1).each do |req|
-          requirement = requirements[req].text.strip
-          requirement_list << requirement
-        end
+          requirement = job_detail_text.partition('Yêu Cầu Công Việc').last.squish.strip
         end
         end
       end
       end
-
-    # other info
+      # benefit
       other_info = job_detail.css('div.content_fck ul li')
       other_info = job_detail.css('div.content_fck ul li')
       (0..other_info.count - 1).each do |part|
       (0..other_info.count - 1).each do |part|
         info = other_info[part].text.squish.strip
         info = other_info[part].text.squish.strip
         other_info_list << info
         other_info_list << info
       end
       end
-
       job = {
       job = {
         title: title,
         title: title,
         company: company,
         company: company,
@@ -87,9 +80,11 @@ namespace :job do
         # position: position,
         # position: position,
         expiration_date: expiration_date,
         expiration_date: expiration_date,
         benefit: benefit_list,
         benefit: benefit_list,
-      description: description_list,
-      requirement: requirement_list,
+        description: description,
+        requirement: requirement,
         other_info: other_info_list
         other_info: other_info_list
       }
       }
+      puts job
+    end
   end
   end
 end
 end