Tô Ngọc Ánh · @anhtn · Thanh Hung Pham · Tô Ngọc Ánh · Thanh Hung Pham · Tô Ngọc Ánh
--- a/lib/tasks/crawler.rake 0 → 100644
+++ b/lib/tasks/crawler.rake 0 → 100644
+task crawl_companies_jobs: :environment do
+  require "open-uri"
--- a/lib/tasks/crawler.rake 0 → 100644
+++ b/lib/tasks/crawler.rake 0 → 100644
+task crawl_companies_jobs: :environment do
+  require "open-uri"
+  crawl_companies_and_jobs(3)
+end
+
+task crawl_industries_locations: :environment do
+  require "open-uri"
+  crawl_industries_and_locations
+end
+
+def crawl_companies_and_jobs(page)
+  for i in 1..page
--- a/lib/tasks/crawler.rake 0 → 100644
+++ b/lib/tasks/crawler.rake 0 → 100644
+    job_description = document.css('')
+  rescue => exception
+    
+  end
+end
+
+def crawl_industries_and_locations
+  document = Nokogiri::HTML(open('https://careerbuilder.vn/viec-lam/tat-ca-viec-lam-vi.html'))
+  industries_xml = document.css('#industry option')
+  industries = industries_xml.map(&:text)
+  locations_xml = document.css('#location option')
+  locations = locations_xml.map(&:text)
+
+
+  industries.each do |industry|
+    exist = Industry.find_by(name: industry).present?
--- a/lib/tasks/crawler.rake 0 → 100644
+++ b/lib/tasks/crawler.rake 0 → 100644
+def crawl_industries_and_locations
+  document = Nokogiri::HTML(open('https://careerbuilder.vn/viec-lam/tat-ca-viec-lam-vi.html'))
+  industries_xml = document.css('#industry option')
+  industries = industries_xml.map(&:text)
+  locations_xml = document.css('#location option')
+  locations = locations_xml.map(&:text)
+
+
+  industries.each do |industry|
+    exist = Industry.find_by(name: industry).present?
+    break if exist
+    puts industry
+    Industry.create!(name: industry)
+  end
+
+  locations.take(70).each do |location|
--- a/lib/tasks/crawler.rake 0 → 100644
+++ b/lib/tasks/crawler.rake 0 → 100644
+
+  [company_links, job_links]
+end
+
+def crawl_companies(company_links)
+  company_links.each do |link|
+    crawl_company(link)
+  end
+end
+
+def crawl_company(company_link)
+  begin
+    document = Nokogiri::HTML(open(company_link))
+    company_name = document.css(".content .name").text
+    exist = Company.find_by(name: company_name).present?
+    return if exist || company_name.empty?
--- a/app/models/location.rb
+++ b/app/models/location.rb
 class Location < ApplicationRecord
+  CITY_VIETNAM_NUMBER = 70
--- a/lib/tasks/crawler.rake 0 → 100644
+++ b/lib/tasks/crawler.rake 0 → 100644
+require "open-uri"
+task crawl_jobs: :environment do
+  job_links = get_job_links(1)
+  crawl_jobs(job_links)
+end
+
+task crawl_industries_locations: :environment do
+  crawl_industries_and_locations
+end
+
+def get_job_links(page)
+  job_links = []
+  page.times do |i|
+    document = Nokogiri::HTML(open("https://careerbuilder.vn/viec-lam/tat-ca-viec-lam-trang-#{i}-vi.html"))
+    jobs_xml = document.xpath('//div/a[@class="job_link"]/@href')
+    jobs_xml.each { |i| job_links << i.value}
--- a/lib/tasks/crawler.rake 0 → 100644
+++ b/lib/tasks/crawler.rake 0 → 100644
+require "open-uri"