Skip to content

V
Venjob_HungNT
Commit a93ab3c6 by Ngô Trung Hưng
Options

fix bug

parent 99841a3b
Showing with 40 additions and 172 deletions
Gemfile
...@@ -22,6 +22,7 @@ gem 'turbolinks', '~> 5' ...@@ -22,6 +22,7 @@ gem 'turbolinks', '~> 5'
gem 'jbuilder', '~> 2.5' gem 'jbuilder', '~> 2.5'
gem 'nokogiri' gem 'nokogiri'
gem 'rubyzip' gem 'rubyzip'
gem 'whenever'
# Use Redis adapter to run Action Cable in production # Use Redis adapter to run Action Cable in production
# gem 'redis', '~> 4.0' # gem 'redis', '~> 4.0'
# Use ActiveModel has_secure_password # Use ActiveModel has_secure_password
......
Gemfile.lock
...@@ -65,6 +65,7 @@ GEM ...@@ -65,6 +65,7 @@ GEM
chromedriver-helper (2.1.1) chromedriver-helper (2.1.1)
archive-zip (~> 0.10) archive-zip (~> 0.10)
nokogiri (~> 1.8) nokogiri (~> 1.8)
chronic (0.10.2)
coderay (1.1.3) coderay (1.1.3)
coffee-rails (4.2.2) coffee-rails (4.2.2)
coffee-script (>= 2.2.0) coffee-script (>= 2.2.0)
...@@ -217,6 +218,8 @@ GEM ...@@ -217,6 +218,8 @@ GEM
websocket-driver (0.7.3) websocket-driver (0.7.3)
websocket-extensions (>= 0.1.0) websocket-extensions (>= 0.1.0)
websocket-extensions (0.1.5) websocket-extensions (0.1.5)
whenever (1.0.0)
chronic (>= 0.6.3)
xpath (3.2.0) xpath (3.2.0)
nokogiri (~> 1.8) nokogiri (~> 1.8)
...@@ -249,6 +252,7 @@ DEPENDENCIES ...@@ -249,6 +252,7 @@ DEPENDENCIES
tzinfo-data tzinfo-data
uglifier (>= 1.3.0) uglifier (>= 1.3.0)
web-console (>= 3.3.0) web-console (>= 3.3.0)
whenever
RUBY VERSION RUBY VERSION
ruby 2.6.6p146 ruby 2.6.6p146
......
app/assets/stylesheets/custom.scss
...@@ -9,7 +9,7 @@ body { ...@@ -9,7 +9,7 @@ body {
} }
$breakpoint-tablet: 992px; $breakpoint-tablet: 992px;
$main-color: #221f20; $main-color: #23303D;
// Header // Header
.cus_header { .cus_header {
height: 52px; height: 52px;
......
config/database.yml
...@@ -14,7 +14,7 @@ default: &default ...@@ -14,7 +14,7 @@ default: &default
encoding: utf8 encoding: utf8
pool: <%= ENV.fetch("RAILS_MAX_THREADS") { 5 } %> pool: <%= ENV.fetch("RAILS_MAX_THREADS") { 5 } %>
username: root username: root
password: '12345678' password: '1'
socket: /var/run/mysqld/mysqld.sock socket: /var/run/mysqld/mysqld.sock
......
config/schedule.rb 0 → 100644
every '53 13 23 7 *' do
rake "crawler:populate"
end
\ No newline at end of file
lib/src/crawler.rb deleted 100644 → 0
require 'open-uri'
require 'src/interface_web'
class Clawler
@@page = Nokogiri::HTML(URI.open('https://careerbuilder.vn/viec-lam/tat-ca-viec-lam-vi.html'))
# PILL DATA CITIES
def self.make_cities
puts "Crawling data location... \n. \n. \n."
data_list_cities = []
data = @@page.search("#location option")
list_cities = data.to_s.split("</option>")
list_cities.each do |x|
data_list_cities << x.gsub(/(^<[\w\D]*>)/, '').gsub(/\n/,'').rstrip
end
puts "Save data to database... \n------------------------"
data_list_cities.each_with_index do |val, index|
area = index > 69 ? 0 : 1
City.find_or_create_by(name: val) do |city|
city.name = val
city.area = area
end
end
end
#PIL DATA INDUSTRIES
def self.make_industries
puts "Crawling data industries... \n. \n. \n."
data_list_industries = []
data = @@page.search("#industry option")
list_industries = data.to_s.split("</option>")
list_industries.each do |x|
data_list_industries << x.gsub(/(^<[\w\D]*>)/, '').gsub(/\n/,'').strip
end
puts "Save data to database... \n------------------------"
data_list_industries.each do |val|
val.gsub!('&amp;','&') if val.include?('&amp;')
Industry.find_or_create_by(name: val) { |industry| industry.name = val }
end
end
# FILL DATA COMPANIES
def self.make_companies
Company.find_or_create_by(name: 'Bảo mật', address: 'Vui lòng xem trong mô tả công việc') do |company|
company.name = 'Bảo mật'
company.address = 'Vui lòng xem trong mô tả công việc'
company.short_description = 'Vui lòng xem trong mô tả công việc'
end
@data = InterfaceWeb.craw_data_companies()
puts 'Save info companies to database . . .'
@data[:name].each_with_index do |name, index|
if Company.find_by(name: name).blank?
address = @data[:address][index]
short_description = @data[:description][index]
Company.create!(name: name,
address: address,
short_description: short_description)
end
end
end
# FILL DATA JOBS
def self.make_jobs
Job.update_all(newdata: 0)
@data_jobs = InterfaceWeb.make_data()
puts 'Save to database . . .'
@data_jobs[:name].each_with_index do |n,i|
name = n.to_s
company_name = @data_jobs[:company_name][i].to_s.strip
id_company = Company.find_by name: company_name
if id_company != nil
id_company = id_company.id
else
id_company = 1
end
level = @data_jobs[:level][i].to_s
experience = @data_jobs[:exprience][i].to_s
salary = @data_jobs[:salary][i].to_s
create_date = @data_jobs[:created_date][i].to_s
expiration_date = @data_jobs[:expiration_date][i].to_s
description = @data_jobs[:description][i].to_s
id_job = Job.create!(name: name,
company_id: id_company,
level: level,
experience: experience,
salary: salary,
create_date: create_date,
expiration_date: expiration_date,
description: description,
newdata: 1)
self.make_foreign_industries_table(@data_jobs[:industry_name][i],id_job.id)
self.make_foreign_cities_table(@data_jobs[:city_name][i],id_job.id)
end
end
def self.make_foreign_industries_table(data,id_job)
@content = data.split(',')
length = @content.length
length.times do |n|
id_industry = Industry.find_by name: (@content[n].strip)
if !id_industry
id_industry = Industry.create!(name: @content[n].strip).id
else
id_industry = id_industry.id
end
IndustryJob.create!(industry_id: id_industry,
job_id: id_job)
end
end
def self.make_foreign_cities_table(data,id_job)
@cities = data.split(',')
# length = @cities.length
@cities.each do |city|
id_cities = City.find_by name: city.strip
if !id_cities
id_cities = City.create!(name: city.strip, area: 1).id
else
id_cities = id_cities.id
end
CityJob.create!(job_id: id_job,
city_id: id_cities)
end
end
end
lib/src/ftp.rb
...@@ -41,12 +41,9 @@ class FtpSever ...@@ -41,12 +41,9 @@ class FtpSever
def self.parse_csv_cities(data) def self.parse_csv_cities(data)
puts 'Import data cities . . .' puts 'Import data cities . . .'
arr_city = '' cities = data['work place'].uniq.select { |val| val.present? }
cities = data['work place'].select { |val| val.present? } cities = cities.map{ |val| val.delete("[]\"") }
cities.uniq! cities.each do |val|
arr_city = cities.map{ |val| val.delete("[]\"") }
arr_city.each do |val|
if !val.blank? if !val.blank?
City.find_or_create_by(name: val) do |city| City.find_or_create_by(name: val) do |city|
city.name = val city.name = val
...@@ -78,11 +75,7 @@ class FtpSever ...@@ -78,11 +75,7 @@ class FtpSever
data['name'].each_with_index do |name, index| data['name'].each_with_index do |name, index|
desc = (data['requirement'][index]).to_s + '\n' << (data['description'][index]).to_s desc = (data['requirement'][index]).to_s + '\n' << (data['description'][index]).to_s
id_company = Company.find_by name: data['company name'][index].to_s.strip id_company = Company.find_by name: data['company name'][index].to_s.strip
if id_company.blank? id_company = id_company.blank? ? 1 : id_company.id
id_company = 1
else
id_company = id_company.id
end
begin begin
id_job = Job.create!( name: name, id_job = Job.create!( name: name,
company_id: id_company, company_id: id_company,
...@@ -104,25 +97,16 @@ class FtpSever ...@@ -104,25 +97,16 @@ class FtpSever
def self.make_foreign_cities_table(data,id_job) def self.make_foreign_cities_table(data,id_job)
data = data.to_s.delete("[]\"") data = data.to_s.delete("[]\"")
id_cities = City.find_by name: data.strip id_city = City.find_by name: data.strip
if id_cities.blank? id_city = id_city.blank? ? City.create!(name: data.strip, area: 1).id : id_city.id
id_cities = City.create!(name: data.strip, area: 1).id CityJob.create!(job_id: id_job, city_id: id_city)
else
id_cities = id_cities.id
end
CityJob.create!(job_id: id_job, city_id: id_cities)
end end
def self.make_foreign_industries_table(data,id_job) def self.make_foreign_industries_table(data,id_job)
data = data.to_s.gsub(',','/').gsub('/',' / ') data = data.to_s.gsub(',','/').gsub('/',' / ')
id_industry = Industry.find_by name: (data.strip) id_industry = Industry.find_by name: (data.strip)
if id_industry.blank? id_industry = id_industry.blank? ? Industry.create!(name: data.strip).id : id_industry.id
id_industry = Industry.create!(name: data.strip).id IndustryJob.create!(industry_id: id_industry, job_id: id_job)
else
id_industry = id_industry.id
end
IndustryJob.create!(industry_id: id_industry,
job_id: id_job)
end end
def self.import_data_from_csv def self.import_data_from_csv
......
lib/tasks/crawler.rake
require 'src/crawler' require 'open-uri'
require 'src/ftp' require 'src/interface_web'
namespace :crawler do namespace :crawler do
task populate: :environment do task populate: :environment do
Clawler.make_industries InterfaceWeb.craw_data_companies()
Clawler.make_cities InterfaceWeb.make_data()
Clawler.make_companies
Clawler.make_jobs
end
task csv: :environment do
Company.find_or_create_by(name: 'Bảo mật', address: 'Vui lòng xem trong mô tả công việc') do |company|
company.name = 'Bảo mật'
company.address = 'Vui lòng xem trong mô tả công việc'
company.short_description = 'Vui lòng xem trong mô tả công việc'
end
FtpSever.import_data_from_csv
end end
end end
lib/tasks/import_csv.rake 0 → 100644
require 'src/ftp'
namespace :csv do
task import_csv: :environment do
Company.find_or_create_by(name: 'Bảo mật', address: 'Vui lòng xem trong mô tả công việc') do |company|
company.name = 'Bảo mật'
company.address = 'Vui lòng xem trong mô tả công việc'
company.short_description = 'Vui lòng xem trong mô tả công việc'
end
FtpSever.import_data_from_csv
end
end
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment