Xuan Trung Le · @trunglx · Ba Toi Dang · Xuan Trung Le · Ba Toi Dang · Xuan Trung Le
--- a/app/data/import.rb 0 → 100644
+++ b/app/data/import.rb 0 → 100644
+require "crawler.rb"
--- a/app/data/crawler.rb 0 → 100644
+++ b/app/data/crawler.rb 0 → 100644
+class Crawler
+  BASE_CAREERBUILDER_URL = "https://careerbuilder.vn"
+  LIST_URL = "#{BASE_CAREERBUILDER_URL}/viec-lam"
+
+  def self.crawl_job_infomation(job_links)
+    links = job_links
+    job_details = []
+
+    links.each do |link|
+      puts "Fetching #{link}..."
+      params = {}
+      link = URI.escape(link)
+      doc = Nokogiri::HTML(open(link), nil, 'utf-8')
+
+      if doc.css('#template_vantai').blank? &&
+        doc.css('#template_1').blank? &&
--- a/lib/tasks/data.rake 0 → 100644
+++ b/lib/tasks/data.rake 0 → 100644
+ENV["RAILS_ENV"] ||= "production"
--- a/db/migrate/20171005085453_create_join_table_cities_companies.rb 0 → 100644
+++ b/db/migrate/20171005085453_create_join_table_cities_companies.rb 0 → 100644
+class CreateJoinTableCitiesCompanies < ActiveRecord::Migration[5.1]
+  def change
+    create_join_table :cities, :companies, id: false do |t|
+      t.index [:city_id, :company_id]
--- a/app/models/job.rb
+++ b/app/models/job.rb
+      job.company.description = item[:company_description]
+      job.company.cities = job.cities
+
+      # Industry
+      unless item[:industry].blank?
+        item[:industry].split(',').each do |name|
+          job.industries << Industry.find_or_create_by(name: name.strip)
+        end
+      end
+      puts "Saving #{item[:name]} ......................................"
+      job.save
+    end
+  end
+
+  def self.filter_link_exist(links)
+    return links - Job.all.map{|job| job.original_link}
--- a/db/migrate/20171004094208_add_original_link_to_jobs.rb 0 → 100644
+++ b/db/migrate/20171004094208_add_original_link_to_jobs.rb 0 → 100644
+class AddOriginalLinkToJobs < ActiveRecord::Migration[5.1]
+  def change
+    add_column :jobs, :original_link, :string
--- a/app/models/job.rb
+++ b/app/models/job.rb
+      unless item[:city].blank?
+        item[:city].split(',').each do |name|
+          job.cities << City.find_or_create_by(name: name.strip)
+        end
+      end
+
+      # Company
+      job.company = Company.find_or_create_by(name: item[:company_name])
+      job.company.location = item[:company_location]
+      job.company.description = item[:company_description]
+      job.company.cities = job.cities
+
+      # Industry
+      unless item[:industry].blank?
+        item[:industry].split(',').each do |name|
+          job.industries << Industry.find_or_create_by(name: name.strip)
--- a/app/models/job.rb
+++ b/app/models/job.rb
+  has_and_belongs_to_many :cities
+
+  def self.create_new_jobs(arr_jobs)
+    arr_jobs.each do |item|
+      job = Job.new(name:          item[:name],
+                    salary:        item[:salary],
+                    description:   item[:description],
+                    level:         item[:level],
+                    original_link: item[:original_link],
+                    experience:    item[:experience],
+                    expiry_date:   item[:expiry_date],
+                    updated_date:  item[:updated_date])
+      # City
+      unless item[:city].blank?
+        item[:city].split(',').each do |name|
+          job.cities << City.find_or_create_by(name: name.strip)
--- a/app/models/job.rb
+++ b/app/models/job.rb
+      end
+
+      # Company
+      job.company = Company.find_or_create_by(name: item[:company_name])
+      job.company.location = item[:company_location]
+      job.company.description = item[:company_description]
+      job.company.cities = job.cities
+
+      # Industry
+      unless item[:industry].blank?
+        item[:industry].split(',').each do |name|
+          job.industries << Industry.find_or_create_by(name: name.strip)
+        end
+      end
+      puts "Saving #{item[:name]} ......................................"
+      job.save
--- a/app/data/crawler.rb 0 → 100644
+++ b/app/data/crawler.rb 0 → 100644
+      description << child.to_html
+    end
+    params[:description] = description.join("")
+
+    # original_link
+    params[:original_link] = link
+
+    return params
+  end
+
+  def self.crawl_company_infomation(doc)
+    # get company information
+    params = {}
+    if doc.css('.box1Detail .TitleDetailNew span').length > 0
+      params[:name] = doc.css('.box1Detail .TitleDetailNew span').text
+      params[:location] = doc.css('.box1Detail .TitleDetailNew label')[0].text
--- a/app/data/crawler.rb 0 → 100644
+++ b/app/data/crawler.rb 0 → 100644
+# encoding: UTF-8
+require 'nokogiri'
+require 'uri'
+require 'open-uri'
+
+class Crawler
+  BASE_CAREERBUILDER_URL = "https://careerbuilder.vn"
+  LIST_URL = "#{BASE_CAREERBUILDER_URL}/viec-lam"
+
+  def self.crawl_job_infomation(job_links)
+    links = job_links
--- a/app/data/crawler.rb 0 → 100644
+++ b/app/data/crawler.rb 0 → 100644
+require 'nokogiri'
+require 'uri'
+require 'open-uri'
+
+class Crawler
+  BASE_CAREERBUILDER_URL = "https://careerbuilder.vn"
+  LIST_URL = "#{BASE_CAREERBUILDER_URL}/viec-lam"
+
+  def self.crawl_job_infomation(job_links)
+    links = job_links
+    job_details = []
+
+    links.each do |link|
+      puts "Fetching #{link}..."
+      params = {}
+      link = URI.escape(link)
--- a/app/data/crawler.rb 0 → 100644
+++ b/app/data/crawler.rb 0 → 100644
+    links.each do |link|
+      puts "Fetching #{link}..."
+      params = {}
+      link = URI.escape(link)
+      doc = Nokogiri::HTML(open(link), nil, 'utf-8')
+
+      if doc.css('#template_vantai, #template_1,
+                  #template_2, #template_3, #template_4,
+                  #template_5, #template_6, #template_7,
+                  #newyear_02').blank?
+
+        params = use_template_default(doc, link)
+        job_details << params
+      end
+    end
+    return job_details
--- a/app/data/crawler.rb 0 → 100644
+++ b/app/data/crawler.rb 0 → 100644
+    links.each do |link|
+      puts "Fetching #{link}..."
+      params = {}
+      link = URI.escape(link)
+      doc = Nokogiri::HTML(open(link), nil, 'utf-8')
+
+      if doc.css('#template_vantai, #template_1,
+                  #template_2, #template_3, #template_4,
+                  #template_5, #template_6, #template_7,
+                  #newyear_02').blank?
+
+        params = use_template_default(doc, link)
+        job_details << params
+      end
+    end
+    return job_details
--- a/app/data/crawler.rb 0 → 100644
+++ b/app/data/crawler.rb 0 → 100644
+
+  def self.crawl_company_infomation(doc)
+    # get company information
+    params = {}
+    if doc.css('.box1Detail .TitleDetailNew span').length > 0
+      params[:name] = doc.css('.box1Detail .TitleDetailNew span').text
+      params[:location] = doc.at('.box1Detail .TitleDetailNew label').text
+      params[:description] = doc.css('.desc_company p').text
+    end
+    params[:name] ||= 'Bảo mật'
+
+    return params
+  end
+
+  def self.get_job_link
+    url = "#{LIST_URL}/tat-ca-viec-lam-trang-#{1}-vi.html"
--- a/app/models/city.rb
+++ b/app/models/city.rb
 class City < ApplicationRecord
-  belongs_to :country
-  has_many :companies
-  has_many :jobs
+  belongs_to :country, optional: true
+  has_and_belongs_to_many :companies
+  has_and_belongs_to_many :jobs
+  def self.list
+      ['AN GIANG', 'BÀ RỊA - VŨNG TÀU', 'BẮC GIANG', 'BẮC KẠN', 'BẠC LIÊU', 'BẮC NINH', 'BẾN TRE',
--- a/app/models/job.rb
+++ b/app/models/job.rb
 class Job < ApplicationRecord
-  belongs_to :city
  belongs_to :company
  has_many :apply_jobs
  has_many :candidates, through: :apply_jobs, class_name: 'User', source: :user
  has_many :favorite_jobs
  has_many :people_who_liked, through: :favorite_jobs, class_name: 'User', source: :user
  has_and_belongs_to_many :industries
+  has_and_belongs_to_many :cities
+
+  def self.create_new_jobs(arr_jobs)
+    viet_nam = Country.find_or_create_by(name: 'Viet Nam')
--- a/app/models/job.rb
+++ b/app/models/job.rb
+    arr_jobs.each do |item|
+      job_cities = []
+      city_name = []
+      job_industries = []
+      industry_name = []
+      job = Job.new(name:          item[:name],
+                    salary:        item[:salary],
+                    description:   item[:description],
+                    level:         item[:level],
+                    original_link: item[:original_link],
+                    experience:    item[:experience],
+                    expiry_date:   item[:expiry_date],
+                    updated_date:  item[:updated_date])
+      # City
+      unless item[:city].blank?
+        city_name = item[:city].split(',').map(&:strip)
--- a/app/data/crawler.rb 0 → 100644
+++ b/app/data/crawler.rb 0 → 100644
+  end
+
+  def self.crawl_company_infomation(doc)
+    # get company information
+    params = {}
+    if doc.css('.box1Detail .TitleDetailNew span').length > 0
+      params[:name] = doc.css('.box1Detail .TitleDetailNew span').text
+      params[:location] = doc.at('.box1Detail .TitleDetailNew label').text
+      params[:description] = doc.css('.desc_company p').text
+    end
+    params[:name] ||= 'Bảo mật'
+
+    return params
+  end
+
+  def self.get_job_link
--- a/app/data/crawler.rb 0 → 100644
+++ b/app/data/crawler.rb 0 → 100644
+  end
+
+  def self.crawl_company_infomation(doc)
+    # get company information
+    params = {}
+    if doc.css('.box1Detail .TitleDetailNew span').length > 0
+      params[:name] = doc.css('.box1Detail .TitleDetailNew span').text
+      params[:location] = doc.at('.box1Detail .TitleDetailNew label').text
+      params[:description] = doc.css('.desc_company p').text
+    end
+    params[:name] ||= 'Bảo mật'
+
+    return params
+  end
+
+  def self.get_job_link
--- a/app/models/job.rb
+++ b/app/models/job.rb
+                    original_link: item[:original_link],
+                    experience:    item[:experience],
+                    expiry_date:   item[:expiry_date],
+                    updated_date:  item[:updated_date])
+      # City
+      unless item[:city].blank?
+        city_names = item[:city].split(',').map(&:strip)
+
+        job_cities = City.where(name: city_names)
+        job_cities.each do |city|
+          job.cities << city
+        end
+      end
+
+      # Company
+      job.company = Company.find_or_create_by(name: item[:company_name])
--- a/app/models/job.rb
+++ b/app/models/job.rb
+                    original_link: item[:original_link],
+                    experience:    item[:experience],
+                    expiry_date:   item[:expiry_date],
+                    updated_date:  item[:updated_date])
+      # City
+      unless item[:city].blank?
+        city_names = item[:city].split(',').map(&:strip)
+
+        job_cities = City.where(name: city_names)
+        job_cities.each do |city|
+          job.cities << city
+        end
+      end
+
+      # Company
+      job.company = Company.find_or_create_by(name: item[:company_name])
--- a/app/models/job.rb
+++ b/app/models/job.rb
+      job_industries = []
+      industry_names = []
+      job = Job.new(name:          item[:name],
+                    salary:        item[:salary],
+                    description:   item[:description],
+                    level:         item[:level],
+                    original_link: item[:original_link],
+                    experience:    item[:experience],
+                    expiry_date:   item[:expiry_date],
+                    updated_date:  item[:updated_date])
+      # City
+      unless item[:city].blank?
+        city_names = item[:city].split(',').map(&:strip)
+
+        job_cities = City.where(name: city_names)
+        job_cities.each do |city|