fix some bugs

lib/tasks/web_crawler.rake

@@ -6,94 +6,98 @@ namespace :crawler do
   desc 'importjob'
 
   task jobs: :environment do
-    parsed_page = Nokogiri::HTML(HTTParty.get('https://careerbuilder.vn/viec-lam/tat-ca-viec-lam-vi.html').body)
-    jobs_item = parsed_page.css('div.job-item .job_link')
-    (0..jobs_item.length - 1).each do |item|
-      job_link = jobs_item[item].attribute('href').text
-      job_page = Nokogiri::HTML(HTTParty.get(job_link).body)
-      job_desc = job_page.css('div.job-desc')
-      job_detail = job_page.css('section.job-detail-content')
-      # title - company
-      title = job_desc.css('h1.title').text
-      company = job_desc.css('a.job-company-name').text
-      # info box
-      info_box = job_detail.css('div.detail-box')
-      info_box_item = info_box.css('ul li')
-      city_box = info_box.css('div.map a')
-      # city, update_at, industry, type, salary, experience, level, expiration_date
-      update_at, industry, type, salary, experience, level, expiration_date = ''
-      city = city_box.text
-      (0..info_box_item.length - 1).each do |part|
-        info = info_box_item[part].text
-        if info.include?(key = 'Ngày cập nhật')
-          update_at = info.squish.remove(key).strip
-        elsif info.include?(key = 'Ngành nghề')
-          industry = info.squish.remove(key).strip
-        elsif info.include?(key = 'Hình thức')
-          type = info.squish.remove(key).strip
-        elsif info.include?(key = 'Lương')
-          salary = info.squish.remove(key).strip
-        elsif info.include?(key = 'Kinh nghiệm')
-          experience = info.squish.remove(key).strip
-        elsif info.include?(key = 'Cấp bậc')
-          level = info.squish.remove(key).strip
-        elsif info.include?(key = 'Hết hạn nộp')
-          expiration_date = info.squish.remove(key).strip
+    (1..3).each do |page|
+      parsed_page = Nokogiri::HTML(HTTParty.get("https://careerbuilder.vn/viec-lam/tat-ca-viec-lam-trang-#{page}-vi.html").body)
+      jobs_item = parsed_page.css('div.job-item .job_link')
+
+      jobs_item.each do |item|
+        job_page = Nokogiri::HTML(HTTParty.get('https://careerbuilder.vn/vi/tim-viec-lam/' +
+          CGI.escape(item.attribute('href').text.remove('https://careerbuilder.vn/vi/tim-viec-lam/'))).body)
+        job_detail = job_page.css('section.job-detail-content')
+        # title - company
+        title = job_page.css('div.job-desc h1.title').text
+        company = job_page.css('div.job-desc a.job-company-name').text
+        # info box
+        info_box_item = job_detail.css('.detail-box ul li')
+        # city, update_at, industry, type, salary, experience, level, expiration_date
+        update_at, industry, type, salary, experience, level, expiration_date = ''
+        city = job_detail.css('.detail-box .map a').text
+
+        info_box_item.each do |info_item|
+          info = info_item.text
+          if info.include?(key = 'Ngày cập nhật')
+            update_at = info.squish.remove(key).strip
+          elsif info.include?(key = 'Ngành nghề')
+            industry = info.squish.remove(key).strip
+          elsif info.include?(key = 'Hình thức')
+            type = info.squish.remove(key).strip
+          elsif info.include?(key = 'Lương')
+            salary = info.squish.remove(key).strip
+          elsif info.include?(key = 'Kinh nghiệm')
+            experience = info.squish.remove(key).strip
+          elsif info.include?(key = 'Cấp bậc')
+            level = info.squish.remove(key).strip
+          elsif info.include?(key = 'Hết hạn nộp')
+            expiration_date = info.squish.remove(key).strip
+          end
         end
-      end
-      # benefit
-      job_detail_row = job_detail.css('div.detail-row')
-      benefit_list = []
-      other_info_list = []
-      benefits = job_detail.css('ul.welfare-list li')
-      (0..benefits.length - 1).each do |part|
-        benefit = benefits[part].text.strip
-        benefit_list << benefit
-      end
-      # description, requirement
-      description, requirement = ''
-      (0..job_detail_row.length - 1).each do |part|
-        job_detail_text = job_detail_row[part].text
-        if job_detail_text.include?('Mô tả Công việc')
-          description = job_detail_text.partition('Mô tả Công việc').last.squish.strip
-        elsif job_detail_text.include?('Yêu Cầu Công Việc')
-          requirement = job_detail_text.partition('Yêu Cầu Công Việc').last.squish.strip
+
+        # benefit
+        benefit_list = []
+        other_info_list = []
+        benefits = job_detail.css('ul.welfare-list li')
+        benefits.each do |part|
+          benefit = part.text.strip
+          benefit_list << benefit
         end
+        # description, requirement
+        description, requirement = ''
+        job_detail_row = job_detail.css('div.detail-row')
+        job_detail_row.each do |part|
+          job_detail_text = part.text
+          if job_detail_text.include?('Mô tả Công việc')
+            description = job_detail_text.partition('Mô tả Công việc').last.squish.strip
+          elsif job_detail_text.include?('Yêu Cầu Công Việc')
+            requirement = job_detail_text.partition('Yêu Cầu Công Việc').last.squish.strip
+          end
+        end
+        # other info
+        other_info = job_detail.css('div.content_fck ul li')
+
+        other_info.each do |part|
+          info = part.text.squish.strip
+          other_info_list << info
+        end
+        job = {
+          title: title,
+          company: company,
+          city: city,
+          update_at: update_at,
+          industry: industry,
+          type: type,
+          salary: salary,
+          experences: experience,
+          level: level,
+          # position: position,
+          expiration_date: expiration_date,
+          benefit: benefit_list,
+          description: description,
+          requirement: requirement,
+          other_info: other_info_list
+        }
+        puts job
       end
-      # benefit
-      other_info = job_detail.css('div.content_fck ul li')
-      (0..other_info.length - 1).each do |part|
-        info = other_info[part].text.squish.strip
-        other_info_list << info
-      end
-      job = {
-        title: title,
-        company: company,
-        city: city,
-        update_at: update_at,
-        industry: industry,
-        type: type,
-        salary: salary,
-        experences: experience,
-        level: level,
-        # position: position,
-        expiration_date: expiration_date,
-        benefit: benefit_list,
-        description: description,
-        requirement: requirement,
-        other_info: other_info_list
-      }
-      puts job
     end
   end
 
   desc 'crawler industry form CareerBuilder'
   task industries: :environment do
-    parsed_page ||= Nokogiri::HTML(HTTParty.get('https://careerbuilder.vn/tim-viec-lam.html').body)
+    parsed_page = Nokogiri::HTML(HTTParty.get('https://careerbuilder.vn/tim-viec-lam.html').body)
     list_job = parsed_page.css('div.list-of-working-positions ul.list-jobs li a')
     industry_list = []
-    (0..list_job.length - 1).each do |part|
-      industry = list_job[part].text.squish.strip
+
+    list_job.each do |part|
+      industry = part.text.squish.strip
       industry_list << industry
     end
     p industry_list
@@ -104,8 +108,9 @@ namespace :crawler do
     parsed_page ||= Nokogiri::HTML(HTTParty.get('https://careerbuilder.vn/tim-viec-lam.html').body)
     list_location = parsed_page.css('div.main-jobs-by-location ul li')
     city_list = []
-    (0..list_location.length - 1).each do |part|
-      city_item = list_location[part].text
+
+    list_location.each do |part|
+      city_item = part.text
       region = 1
       if city_item.include?(key = 'Việc làm tại')
         city_item = city_item.remove(key).strip