Nguyen Hoang Mai Phuong · @phuongnhm · Thanh Hung Pham · Nguyen Hoang Mai Phuong · Thanh Hung Pham · Nguyen Hoang Mai Phuong
--- a/craw.rb 0 → 100644
+++ b/craw.rb 0 → 100644
+require 'nokogiri'
--- a/lib/tasks/crawler.rake 0 → 100644
+++ b/lib/tasks/crawler.rake 0 → 100644
+require 'open-uri'
+namespace :crawler do
--- a/lib/tasks/crawler.rake 0 → 100644
+++ b/lib/tasks/crawler.rake 0 → 100644
+require 'open-uri'
+namespace :crawler do
+  desc "TODO"
--- a/lib/tasks/crawler.rake 0 → 100644
+++ b/lib/tasks/crawler.rake 0 → 100644
+require 'open-uri'
+namespace :crawler do
+  desc "TODO"
+  task jobs: :environment do
+    base_url = Nokogiri::HTML(URI.open('https://careerbuilder.vn/'))
+    list_url = base_url.css('div.menu div.dropdown-menu ul li a')[0].attributes["href"].value
--- a/lib/tasks/crawler.rake 0 → 100644
+++ b/lib/tasks/crawler.rake 0 → 100644
+require 'open-uri'
+namespace :crawler do
+  desc "TODO"
+  task jobs: :environment do
+    base_url = Nokogiri::HTML(URI.open('https://careerbuilder.vn/'))
+    list_url = base_url.css('div.menu div.dropdown-menu ul li a')[0].attributes["href"].value
+    parse_list_url = Nokogiri::HTML(URI.open(list_url))
+    job_listing = parse_list_url.css('div.job-item')
+    page = 1
+    per_page = job_listing.length 
--- a/lib/tasks/crawler.rake 0 → 100644
+++ b/lib/tasks/crawler.rake 0 → 100644
+require 'open-uri'
+namespace :crawler do
+  desc "TODO"
+  task jobs: :environment do
+    base_url = Nokogiri::HTML(URI.open('https://careerbuilder.vn/'))
+    list_url = base_url.css('div.menu div.dropdown-menu ul li a')[0].attributes["href"].value
+    parse_list_url = Nokogiri::HTML(URI.open(list_url))
+    job_listing = parse_list_url.css('div.job-item')
+    page = 1
+    per_page = job_listing.length 
+    total = parse_list_url.css('div.job-found p').text.split(' ')[0].gsub(',','').to_i
+    last_page = (total.to_f / per_page.to_f).round
+    while page <= last_page
+      pagination_list_url = "https://careerbuilder.vn/viec-lam/tat-ca-viec-lam-trang-#{page}-vi.html"
+      parse_list_url = Nokogiri::HTML(URI.open(pagination_list_url))
+      pagination_job_listing = parse_list_url.css('div.job-item')
--- a/lib/tasks/crawler.rake 0 → 100644
+++ b/lib/tasks/crawler.rake 0 → 100644
+require 'open-uri'
+namespace :crawler do
+  desc "TODO"
+  task jobs: :environment do
+    base_url = Nokogiri::HTML(URI.open('https://careerbuilder.vn/'))
+    list_url = base_url.css('div.menu div.dropdown-menu ul li a')[0].attributes["href"].value
+    parse_list_url = Nokogiri::HTML(URI.open(list_url))
+    job_listing = parse_list_url.css('div.job-item')
+    page = 1
+    per_page = job_listing.length 
+    total = parse_list_url.css('div.job-found p').text.split(' ')[0].gsub(',','').to_i
+    last_page = (total.to_f / per_page.to_f).round
--- a/lib/tasks/crawler.rake 0 → 100644
+++ b/lib/tasks/crawler.rake 0 → 100644
+  desc "TODO"
+  task jobs: :environment do
+    base_url = Nokogiri::HTML(URI.open('https://careerbuilder.vn/'))
+    list_url = base_url.css('div.menu div.dropdown-menu ul li a')[0].attributes["href"].value
+    parse_list_url = Nokogiri::HTML(URI.open(list_url))
+    job_listing = parse_list_url.css('div.job-item')
+    page = 1
+    per_page = job_listing.length 
+    total = parse_list_url.css('div.job-found p').text.split(' ')[0].gsub(',','').to_i
+    last_page = (total.to_f / per_page.to_f).round
+    while page <= last_page
+      pagination_list_url = "https://careerbuilder.vn/viec-lam/tat-ca-viec-lam-trang-#{page}-vi.html"
+      parse_list_url = Nokogiri::HTML(URI.open(pagination_list_url))
+      pagination_job_listing = parse_list_url.css('div.job-item')
+      pagination_job_listing.each do |detail_jobs|
+        pagination_url = detail_jobs.css('a')[1].attributes["href"].value
--- a/lib/tasks/crawler.rake 0 → 100644
+++ b/lib/tasks/crawler.rake 0 → 100644
+                expired_at = title_strong.css('p').text.gsub(/\s+/, " ").strip
+              end
+          end      
+        h3_element_value = pagination_detail_job.css('div.detail-row')
+          h3_element_value.each do |h3_element|
+            case h3_element.css('h3').text
+              when "Mô tả Công việc"
+                overview = h3_element.css('p').text.gsub(/\s+/, " ").strip
+              when "Yêu Cầu Công Việc"
+                requirement = h3_element.css('p').text.gsub(/\s+/, " ").strip
+              when "Thông tin khác"
+                other_requirement = h3_element.css('div.content_fck ul li').text.gsub(/\s+/, " ").strip
+              end  
+          end  
+          
+        company_url = detail_jobs.css('a')[0].attributes["href"].value
--- a/lib/tasks/crawler.rake 0 → 100644
+++ b/lib/tasks/crawler.rake 0 → 100644
+  task jobs: :environment do
+    base_url = Nokogiri::HTML(URI.open('https://careerbuilder.vn/'))
+    list_url = base_url.css('div.menu div.dropdown-menu ul li a')[0].attributes["href"].value
+    parse_list_url = Nokogiri::HTML(URI.open(list_url))
+    job_listing = parse_list_url.css('div.job-item')
+    page = 1
+    per_page = job_listing.length 
+    total = parse_list_url.css('div.job-found p').text.split(' ')[0].gsub(',','').to_i
+    last_page = (total.to_f / per_page.to_f).round
+    while page <= last_page
+      pagination_list_url = "https://careerbuilder.vn/viec-lam/tat-ca-viec-lam-trang-#{page}-vi.html"
+      parse_list_url = Nokogiri::HTML(URI.open(pagination_list_url))
+      pagination_job_listing = parse_list_url.css('div.job-item')
+      pagination_job_listing.each do |detail_jobs|
+        pagination_url = detail_jobs.css('a')[1].attributes["href"].value
+        parse_pagination_url = Nokogiri::HTML(URI.open(pagination_url))
--- a/lib/tasks/crawler.rake 0 → 100644
+++ b/lib/tasks/crawler.rake 0 → 100644
+    list_url = base_url.css('div.menu div.dropdown-menu ul li a')[0].attributes["href"].value
+    parse_list_url = Nokogiri::HTML(URI.open(list_url))
+    job_listing = parse_list_url.css('div.job-item')
+    page = 1
+    per_page = job_listing.length 
+    total = parse_list_url.css('div.job-found p').text.split(' ')[0].gsub(',','').to_i
+    last_page = (total.to_f / per_page.to_f).round
+    while page <= last_page
+      pagination_list_url = "https://careerbuilder.vn/viec-lam/tat-ca-viec-lam-trang-#{page}-vi.html"
+      parse_list_url = Nokogiri::HTML(URI.open(pagination_list_url))
+      pagination_job_listing = parse_list_url.css('div.job-item')
+      pagination_job_listing.each do |detail_jobs|
+        pagination_url = detail_jobs.css('a')[1].attributes["href"].value
+        parse_pagination_url = Nokogiri::HTML(URI.open(pagination_url))
+        pagination_detail_job = parse_pagination_url.css('div.container')
+        strong_element_value = pagination_detail_job.css('div.detail-box.has-background ul li')
--- a/lib/tasks/crawler.rake 0 → 100644
+++ b/lib/tasks/crawler.rake 0 → 100644
+    job_listing = parse_list_url.css('div.job-item')
+    page = 1
+    per_page = job_listing.length 
+    total = parse_list_url.css('div.job-found p').text.split(' ')[0].gsub(',','').to_i
+    last_page = (total.to_f / per_page.to_f).round
+    while page <= last_page
+      pagination_list_url = "https://careerbuilder.vn/viec-lam/tat-ca-viec-lam-trang-#{page}-vi.html"
+      parse_list_url = Nokogiri::HTML(URI.open(pagination_list_url))
+      pagination_job_listing = parse_list_url.css('div.job-item')
+      pagination_job_listing.each do |detail_jobs|
+        pagination_url = detail_jobs.css('a')[1].attributes["href"].value
+        parse_pagination_url = Nokogiri::HTML(URI.open(pagination_url))
+        pagination_detail_job = parse_pagination_url.css('div.container')
+        strong_element_value = pagination_detail_job.css('div.detail-box.has-background ul li')
+        title = pagination_detail_job.css('div.job-desc h1.title')[0].text
+          strong_element_value.each do |title_strong|
--- a/lib/tasks/crawler.rake 0 → 100644
+++ b/lib/tasks/crawler.rake 0 → 100644
+              when "Hết hạn nộp"
+                expired_at = title_strong.css('p').text.gsub(/\s+/, " ").strip
+              end
+          end      
+        h3_element_value = pagination_detail_job.css('div.detail-row')
+          h3_element_value.each do |h3_element|
+            case h3_element.css('h3').text
+              when "Mô tả Công việc"
+                overview = h3_element.css('p').text.gsub(/\s+/, " ").strip
+              when "Yêu Cầu Công Việc"
+                requirement = h3_element.css('p').text.gsub(/\s+/, " ").strip
+              when "Thông tin khác"
+                other_requirement = h3_element.css('div.content_fck ul li').text.gsub(/\s+/, " ").strip
+              end  
+          end  
+          
--- a/lib/tasks/crawler.rake 0 → 100644
+++ b/lib/tasks/crawler.rake 0 → 100644
+                overview = h3_element.css('p').text.gsub(/\s+/, " ").strip
+              when "Yêu Cầu Công Việc"
+                requirement = h3_element.css('p').text.gsub(/\s+/, " ").strip
+              when "Thông tin khác"
+                other_requirement = h3_element.css('div.content_fck ul li').text.gsub(/\s+/, " ").strip
+              end  
+          end  
+          
+        company_url = detail_jobs.css('a')[0].attributes["href"].value
+        parse_company_url = Nokogiri::HTML(URI.open(company_url))
+        company = parse_company_url.css('div.container')
+        comapny = company.css('div.company-info div.info div.content p.name').text
+        company_info = company.css('div.company-info div.info div.content')
+        address company_info.css('p')[1].text 
+        description = company_info.css('ul li').text
+        overview = company.css('div.row div.content p').text.gsub(/\s+/, " ").strip
--- a/lib/tasks/crawler.rake 0 → 100644
+++ b/lib/tasks/crawler.rake 0 → 100644
+        comapny = company.css('div.company-info div.info div.content p.name').text
+        company_info = company.css('div.company-info div.info div.content')
+        address company_info.css('p')[1].text 
+        description = company_info.css('ul li').text
+        overview = company.css('div.row div.content p').text.gsub(/\s+/, " ").strip
+      end  
+      page +=1 
+    end 
+
+  end
+
+  desc "TODO"
+  task industries: :environment do
+    industries_listing = parse_base_url.css('div.container div.list-of-working-positions div.col-md-6.col-lg-4.cus-col')
+    industries_listing.each do |industries| 
+      industries_name = industries.css('ul.list-jobs li').text
--- a/lib/tasks/crawler.rake 0 → 100644
+++ b/lib/tasks/crawler.rake 0 → 100644
+    cities_foreign.each do |city|
+      city_name = city.text
+      City.find_or_create_by(
+        name: city_name        
+      )
+    Region.find_or_create_by(name: 'Nước Ngoài').id
+    puts 'Added: ' + (city_name ? city_name : '')
+    end
+  end
+
+
+  desc "TODO"
+  task regions: :environment do
+  regions = parse_base_url.css('div.container div.col-xl-3 div.main-jobs-by-location h3')
+    regions.each do |region|
+      region_name region.text.gsub('Việc Làm','')
--- a/lib/tasks/crawler.rake 0 → 100644
+++ b/lib/tasks/crawler.rake 0 → 100644
+  task industries: :environment do
+    industries_listing = parse_base_url.css('div.container div.list-of-working-positions div.col-md-6.col-lg-4.cus-col')
+    industries_listing.each do |industries| 
+      industries_name = industries.css('ul.list-jobs li').text
+      puts 'Added: ' + (industries_name ? industries_name : '')
+    end
+    
+  end
+
+  desc "TODO"
+  task cities: :environment do
+    cities = parse_base_url.css('div.container div.col-xl-3 div.main-jobs-by-location div.jobs-in-country li a')
+    cities.each do |city|
+      city_name = city.text.gsub('Việc làm tại','')
+      City.find_or_create_by(
+        name: city_name        
--- a/db/migrate/20210720145646_change_jobs.rb 0 → 100644
+++ b/db/migrate/20210720145646_change_jobs.rb 0 → 100644
+class ChangeJobs < ActiveRecord::Migration[6.1]
+  def up
+    change_column :jobs, :salary, :string
+    remove_column :jobs, :industries_type, :text
--- a/lib/tasks/crawler.rake 0 → 100644
+++ b/lib/tasks/crawler.rake 0 → 100644
+require 'open-uri'
+
+namespace :crawler do
+  desc 'Crawl Jobs and Companies'
--- a/lib/tasks/crawler.rake 0 → 100644
+++ b/lib/tasks/crawler.rake 0 → 100644
+    base_url = Nokogiri::HTML(URI.open('https://careerbuilder.vn/'))
+    job_page = base_url.css('div.menu div.dropdown-menu ul li a')[0].attributes['href'].value
+    parse_job_page = Nokogiri::HTML(URI.open(job_page))
+    job_listing = parse_job_page.css('div.job-item')
+    per_page = job_listing.present? ? job_listing.length : 0
+    page = 1
+    total = parse_job_page.css('div.job-found p').text.split(' ')[0].gsub(',', '').to_i
+    last_page = (total.to_f / per_page).round
+
+    while page <= last_page
+      pagination_page_job = "https://careerbuilder.vn/viec-lam/tat-ca-viec-lam-trang-#{page}-vi.html"
+      parse_pagination_page_job = Nokogiri::HTML(URI.open(pagination_page_job))
+      pagination_job_listing = parse_pagination_page_job.css('div.job-item')
+      pagination_job_listing.each do |detail_jobs|
+        company_url = detail_jobs.css('a.company-name').attribute('href').text
+        next if company_url == 'javascript:void(0);'
--- a/lib/tasks/crawler.rake 0 → 100644
+++ b/lib/tasks/crawler.rake 0 → 100644
+    job_listing = parse_job_page.css('div.job-item')
+    per_page = job_listing.present? ? job_listing.length : 0
+    page = 1
+    total = parse_job_page.css('div.job-found p').text.split(' ')[0].gsub(',', '').to_i
+    last_page = (total.to_f / per_page).round
+
+    while page <= last_page
+      pagination_page_job = "https://careerbuilder.vn/viec-lam/tat-ca-viec-lam-trang-#{page}-vi.html"
+      parse_pagination_page_job = Nokogiri::HTML(URI.open(pagination_page_job))
+      pagination_job_listing = parse_pagination_page_job.css('div.job-item')
+      pagination_job_listing.each do |detail_jobs|
+        company_url = detail_jobs.css('a.company-name').attribute('href').text
+        next if company_url == 'javascript:void(0);'
+        slug_company = CGI.escape(company_url.gsub('https://careerbuilder.vn/vi/nha-tuyen-dung/', '').strip)
+        company_page = "https://careerbuilder.vn/vi/nha-tuyen-dung/#{slug_company}"
+        puts company_page
--- a/lib/tasks/crawler.rake 0 → 100644
+++ b/lib/tasks/crawler.rake 0 → 100644
+    last_page = (total.to_f / per_page).round
+
+    while page <= last_page
+      pagination_page_job = "https://careerbuilder.vn/viec-lam/tat-ca-viec-lam-trang-#{page}-vi.html"
+      parse_pagination_page_job = Nokogiri::HTML(URI.open(pagination_page_job))
+      pagination_job_listing = parse_pagination_page_job.css('div.job-item')
+      pagination_job_listing.each do |detail_jobs|
+        company_url = detail_jobs.css('a.company-name').attribute('href').text
+        next if company_url == 'javascript:void(0);'
+        slug_company = CGI.escape(company_url.gsub('https://careerbuilder.vn/vi/nha-tuyen-dung/', '').strip)
+        company_page = "https://careerbuilder.vn/vi/nha-tuyen-dung/#{slug_company}"
+        puts company_page
+        parse_company_page = Nokogiri::HTML(URI.open(company_page))
+        company = parse_company_page.css('div.container')
+        company_name = company.css('div.company-info div.content p.name')
+        next if company_name.nil?
--- a/lib/tasks/crawler.rake 0 → 100644
+++ b/lib/tasks/crawler.rake 0 → 100644
+
+    while page <= last_page
+      pagination_page_job = "https://careerbuilder.vn/viec-lam/tat-ca-viec-lam-trang-#{page}-vi.html"
+      parse_pagination_page_job = Nokogiri::HTML(URI.open(pagination_page_job))
+      pagination_job_listing = parse_pagination_page_job.css('div.job-item')
+      pagination_job_listing.each do |detail_jobs|
+        company_url = detail_jobs.css('a.company-name').attribute('href').text
+        next if company_url == 'javascript:void(0);'
+        slug_company = CGI.escape(company_url.gsub('https://careerbuilder.vn/vi/nha-tuyen-dung/', '').strip)
+        company_page = "https://careerbuilder.vn/vi/nha-tuyen-dung/#{slug_company}"
+        puts company_page
+        parse_company_page = Nokogiri::HTML(URI.open(company_page))
+        company = parse_company_page.css('div.container')
+        company_name = company.css('div.company-info div.content p.name')
+        next if company_name.nil?
+        name = company.css('div.company-info div.content p.name').text
--- a/lib/tasks/crawler.rake 0 → 100644
+++ b/lib/tasks/crawler.rake 0 → 100644
+        company_url = detail_jobs.css('a.company-name').attribute('href').text
+        next if company_url == 'javascript:void(0);'
+        slug_company = CGI.escape(company_url.gsub('https://careerbuilder.vn/vi/nha-tuyen-dung/', '').strip)
+        company_page = "https://careerbuilder.vn/vi/nha-tuyen-dung/#{slug_company}"
+        puts company_page
+        parse_company_page = Nokogiri::HTML(URI.open(company_page))
+        company = parse_company_page.css('div.container')
+        company_name = company.css('div.company-info div.content p.name')
+        next if company_name.nil?
+        name = company.css('div.company-info div.content p.name').text
+        company_info = company.css('div.company-info div.content')
+        address = company_info.css('p')[1].try(:text)
+        description = company_info.css('ul li').text
+        overview = company.css('div.row div.content p').text.squish.strip
+        Company.find_or_create_by(
+          name: name,
--- a/lib/tasks/crawler.rake 0 → 100644
+++ b/lib/tasks/crawler.rake 0 → 100644
+        name = company.css('div.company-info div.content p.name').text
+        company_info = company.css('div.company-info div.content')
+        address = company_info.css('p')[1].try(:text)
+        description = company_info.css('ul li').text
+        overview = company.css('div.row div.content p').text.squish.strip
+        Company.find_or_create_by(
+          name: name,
+          address: address,
+          description: description,
+          overview: overview
+        )
+
+        slug_job = CGI.escape(detail_jobs.css('a.job_link').attribute('href').text
+        .gsub('https://careerbuilder.vn/vi/tim-viec-lam/', '').strip)
+        job_detail_page = "https://careerbuilder.vn/vi/tim-viec-lam/#{slug_job}"
+        puts job_detail_page
--- a/lib/tasks/crawler.rake 0 → 100644
+++ b/lib/tasks/crawler.rake 0 → 100644
+        overview = company.css('div.row div.content p').text.squish.strip
+        Company.find_or_create_by(
+          name: name,
+          address: address,
+          description: description,
+          overview: overview
+        )
+
+        slug_job = CGI.escape(detail_jobs.css('a.job_link').attribute('href').text
+        .gsub('https://careerbuilder.vn/vi/tim-viec-lam/', '').strip)
+        job_detail_page = "https://careerbuilder.vn/vi/tim-viec-lam/#{slug_job}"
+        puts job_detail_page
+        parse_job_detail_page = Nokogiri::HTML(URI.open(job_detail_page))
+        detail_job = parse_job_detail_page.css('div.container')
+        title = detail_job.css('div.job-desc h1.title')
+        next if title.nil?
--- a/lib/tasks/crawler.rake 0 → 100644
+++ b/lib/tasks/crawler.rake 0 → 100644
+        Company.find_or_create_by(
+          name: name,
+          address: address,
+          description: description,
+          overview: overview
+        )
+
+        slug_job = CGI.escape(detail_jobs.css('a.job_link').attribute('href').text
+        .gsub('https://careerbuilder.vn/vi/tim-viec-lam/', '').strip)
+        job_detail_page = "https://careerbuilder.vn/vi/tim-viec-lam/#{slug_job}"
+        puts job_detail_page
+        parse_job_detail_page = Nokogiri::HTML(URI.open(job_detail_page))
+        detail_job = parse_job_detail_page.css('div.container')
+        title = detail_job.css('div.job-desc h1.title')
+        next if title.nil?
+        title_job = detail_job.css('div.job-desc h1.title').text
--- a/lib/tasks/crawler.rake 0 → 100644
+++ b/lib/tasks/crawler.rake 0 → 100644
+        detail_require = detail_job.css('div.detail-row')
+        detail_require.each do |detail|
+          case detail.css('h3').text
+          when 'Phúc lợi '
+            benefits = detail.css('ul li').text
+          when 'Mô tả Công việc'
+            overview = detail.css('p').text
+          when 'Yêu Cầu Công Việc'
+            requirement = detail.css('p').text
+          when 'Thông tin khác'
+            other_requirement = detail.css('div.content_fck ul li').text.squish
+          end
+        end
+
+        job = Job.find_or_create_by(
+          title: title_job,
--- a/lib/tasks/crawler.rake 0 → 100644
+++ b/lib/tasks/crawler.rake 0 → 100644
+            other_requirement = detail.css('div.content_fck ul li').text.squish
+          end
+        end
+
+        job = Job.find_or_create_by(
+          title: title_job,
+          salary: salary,
+          experience: experience,
+          type: type,
+          level: level,
+          expired_at: expired_at,
+          benefits: benefits,
+          overview: overview,
+          requirement: requirement,
+          other_requirement: other_requirement,
+          company_id: Company.find_by(name: name).id
--- a/lib/tasks/crawler.rake 0 → 100644
+++ b/lib/tasks/crawler.rake 0 → 100644
+          title: title_job,
+          salary: salary,
+          experience: experience,
+          type: type,
+          level: level,
+          expired_at: expired_at,
+          benefits: benefits,
+          overview: overview,
+          requirement: requirement,
+          other_requirement: other_requirement,
+          company_id: Company.find_by(name: name).id
+        )
+
+        industries = detail_job.css('div.detail-box.has-background ul li p a')
+        industries.each do |industry|
+        industry_name = industry.text.squish
--- a/lib/tasks/crawler.rake 0 → 100644
+++ b/lib/tasks/crawler.rake 0 → 100644
+          salary: salary,
+          experience: experience,
+          type: type,
+          level: level,
+          expired_at: expired_at,
+          benefits: benefits,
+          overview: overview,
+          requirement: requirement,
+          other_requirement: other_requirement,
+          company_id: Company.find_by(name: name).id
+        )
+
+        industries = detail_job.css('div.detail-box.has-background ul li p a')
+        industries.each do |industry|
+        industry_name = industry.text.squish
+        industries = Industry.find_or_create_by(
--- a/lib/tasks/crawler.rake 0 → 100644
+++ b/lib/tasks/crawler.rake 0 → 100644
+          requirement: requirement,
+          other_requirement: other_requirement,
+          company_id: Company.find_by(name: name).id
+        )
+
+        industries = detail_job.css('div.detail-box.has-background ul li p a')
+        industries.each do |industry|
+        industry_name = industry.text.squish
+        industries = Industry.find_or_create_by(
+          name: industry_name
+        )
+        job.industries << industries
+        end
+
+        location = detail_job.css('div.map p a')
+        location.each do |city|