Skip to content

V
Venjob_HungNT
Commit e806339f by Hung0326 Committed by GitHub
Options

Merge pull request #12 from Hung0326/dev 

fix bug
parents 99841a3b a93ab3c6
Pipeline #693 canceled with stages
in 0 seconds
Showing with 38 additions and 170 deletions
Gemfile
......@@ -22,6 +22,7 @@ gem 'turbolinks', '~> 5'
gem 'jbuilder', '~> 2.5'
gem 'nokogiri'
gem 'rubyzip'
gem 'whenever'
# Use Redis adapter to run Action Cable in production
# gem 'redis', '~> 4.0'
# Use ActiveModel has_secure_password
......
Gemfile.lock
......@@ -65,6 +65,7 @@ GEM
chromedriver-helper (2.1.1)
archive-zip (~> 0.10)
nokogiri (~> 1.8)
chronic (0.10.2)
coderay (1.1.3)
coffee-rails (4.2.2)
coffee-script (>= 2.2.0)
......@@ -217,6 +218,8 @@ GEM
websocket-driver (0.7.3)
websocket-extensions (>= 0.1.0)
websocket-extensions (0.1.5)
whenever (1.0.0)
chronic (>= 0.6.3)
xpath (3.2.0)
nokogiri (~> 1.8)
......@@ -249,6 +252,7 @@ DEPENDENCIES
tzinfo-data
uglifier (>= 1.3.0)
web-console (>= 3.3.0)
whenever
RUBY VERSION
ruby 2.6.6p146
......
app/assets/stylesheets/custom.scss
......@@ -9,7 +9,7 @@ body {
}
$breakpoint-tablet: 992px;
$main-color: #221f20;
$main-color: #23303D;
// Header
.cus_header {
height: 52px;
......
config/database.yml
......@@ -14,7 +14,7 @@ default: &default
encoding: utf8
pool: <%= ENV.fetch("RAILS_MAX_THREADS") { 5 } %>
username: root
password: '12345678'
password: '1'
socket: /var/run/mysqld/mysqld.sock
......
config/schedule.rb 0 → 100644
every '53 13 23 7 *' do
rake "crawler:populate"
end
\ No newline at end of file
lib/src/crawler.rb deleted 100644 → 0
require 'open-uri'
require 'src/interface_web'
class Clawler
@@page = Nokogiri::HTML(URI.open('https://careerbuilder.vn/viec-lam/tat-ca-viec-lam-vi.html'))
# PILL DATA CITIES
def self.make_cities
puts "Crawling data location... \n. \n. \n."
data_list_cities = []
data = @@page.search("#location option")
list_cities = data.to_s.split("</option>")
list_cities.each do |x|
data_list_cities << x.gsub(/(^<[\w\D]*>)/, '').gsub(/\n/,'').rstrip
end
puts "Save data to database... \n------------------------"
data_list_cities.each_with_index do |val, index|
area = index > 69 ? 0 : 1
City.find_or_create_by(name: val) do |city|
city.name = val
city.area = area
end
end
end
#PIL DATA INDUSTRIES
def self.make_industries
puts "Crawling data industries... \n. \n. \n."
data_list_industries = []
data = @@page.search("#industry option")
list_industries = data.to_s.split("</option>")
list_industries.each do |x|
data_list_industries << x.gsub(/(^<[\w\D]*>)/, '').gsub(/\n/,'').strip
end
puts "Save data to database... \n------------------------"
data_list_industries.each do |val|
val.gsub!('&amp;','&') if val.include?('&amp;')
Industry.find_or_create_by(name: val) { |industry| industry.name = val }
end
end
# FILL DATA COMPANIES
def self.make_companies
Company.find_or_create_by(name: 'Bảo mật', address: 'Vui lòng xem trong mô tả công việc') do |company|
company.name = 'Bảo mật'
company.address = 'Vui lòng xem trong mô tả công việc'
company.short_description = 'Vui lòng xem trong mô tả công việc'
end
@data = InterfaceWeb.craw_data_companies()
puts 'Save info companies to database . . .'
@data[:name].each_with_index do |name, index|
if Company.find_by(name: name).blank?
address = @data[:address][index]
short_description = @data[:description][index]
Company.create!(name: name,
address: address,
short_description: short_description)
end
end
end
# FILL DATA JOBS
def self.make_jobs
Job.update_all(newdata: 0)
@data_jobs = InterfaceWeb.make_data()
puts 'Save to database . . .'
@data_jobs[:name].each_with_index do |n,i|
name = n.to_s
company_name = @data_jobs[:company_name][i].to_s.strip
id_company = Company.find_by name: company_name
if id_company != nil
id_company = id_company.id
else
id_company = 1
end
level = @data_jobs[:level][i].to_s
experience = @data_jobs[:exprience][i].to_s
salary = @data_jobs[:salary][i].to_s
create_date = @data_jobs[:created_date][i].to_s
expiration_date = @data_jobs[:expiration_date][i].to_s
description = @data_jobs[:description][i].to_s
id_job = Job.create!(name: name,
company_id: id_company,
level: level,
experience: experience,
salary: salary,
create_date: create_date,
expiration_date: expiration_date,
description: description,
newdata: 1)
self.make_foreign_industries_table(@data_jobs[:industry_name][i],id_job.id)
self.make_foreign_cities_table(@data_jobs[:city_name][i],id_job.id)
end
end
def self.make_foreign_industries_table(data,id_job)
@content = data.split(',')
length = @content.length
length.times do |n|
id_industry = Industry.find_by name: (@content[n].strip)
if !id_industry
id_industry = Industry.create!(name: @content[n].strip).id
else
id_industry = id_industry.id
end
IndustryJob.create!(industry_id: id_industry,
job_id: id_job)
end
end
def self.make_foreign_cities_table(data,id_job)
@cities = data.split(',')
# length = @cities.length
@cities.each do |city|
id_cities = City.find_by name: city.strip
if !id_cities
id_cities = City.create!(name: city.strip, area: 1).id
else
id_cities = id_cities.id
end
CityJob.create!(job_id: id_job,
city_id: id_cities)
end
end
end
lib/src/ftp.rb
......@@ -41,12 +41,9 @@ class FtpSever
def self.parse_csv_cities(data)
puts 'Import data cities . . .'
arr_city = ''
cities = data['work place'].select { |val| val.present? }
cities.uniq!
arr_city = cities.map{ |val| val.delete("[]\"") }
arr_city.each do |val|
cities = data['work place'].uniq.select { |val| val.present? }
cities = cities.map{ |val| val.delete("[]\"") }
cities.each do |val|
if !val.blank?
City.find_or_create_by(name: val) do |city|
city.name = val
......@@ -78,11 +75,7 @@ class FtpSever
data['name'].each_with_index do |name, index|
desc = (data['requirement'][index]).to_s + '\n' << (data['description'][index]).to_s
id_company = Company.find_by name: data['company name'][index].to_s.strip
if id_company.blank?
id_company = 1
else
id_company = id_company.id
end
id_company = id_company.blank? ? 1 : id_company.id
begin
id_job = Job.create!( name: name,
company_id: id_company,
......@@ -104,25 +97,16 @@ class FtpSever
def self.make_foreign_cities_table(data,id_job)
data = data.to_s.delete("[]\"")
id_cities = City.find_by name: data.strip
if id_cities.blank?
id_cities = City.create!(name: data.strip, area: 1).id
else
id_cities = id_cities.id
end
CityJob.create!(job_id: id_job, city_id: id_cities)
id_city = City.find_by name: data.strip
id_city = id_city.blank? ? City.create!(name: data.strip, area: 1).id : id_city.id
CityJob.create!(job_id: id_job, city_id: id_city)
end
def self.make_foreign_industries_table(data,id_job)
data = data.to_s.gsub(',','/').gsub('/',' / ')
id_industry = Industry.find_by name: (data.strip)
if id_industry.blank?
id_industry = Industry.create!(name: data.strip).id
else
id_industry = id_industry.id
end
IndustryJob.create!(industry_id: id_industry,
job_id: id_job)
id_industry = id_industry.blank? ? Industry.create!(name: data.strip).id : id_industry.id
IndustryJob.create!(industry_id: id_industry, job_id: id_job)
end
def self.import_data_from_csv
......
lib/tasks/crawler.rake
require 'src/crawler'
require 'src/ftp'
require 'open-uri'
require 'src/interface_web'
namespace :crawler do
task populate: :environment do
Clawler.make_industries
Clawler.make_cities
Clawler.make_companies
Clawler.make_jobs
end
task csv: :environment do
Company.find_or_create_by(name: 'Bảo mật', address: 'Vui lòng xem trong mô tả công việc') do |company|
company.name = 'Bảo mật'
company.address = 'Vui lòng xem trong mô tả công việc'
company.short_description = 'Vui lòng xem trong mô tả công việc'
end
FtpSever.import_data_from_csv
InterfaceWeb.craw_data_companies()
InterfaceWeb.make_data()
end
end
lib/tasks/import_csv.rake 0 → 100644
require 'src/ftp'
namespace :csv do
task import_csv: :environment do
Company.find_or_create_by(name: 'Bảo mật', address: 'Vui lòng xem trong mô tả công việc') do |company|
company.name = 'Bảo mật'
company.address = 'Vui lòng xem trong mô tả công việc'
company.short_description = 'Vui lòng xem trong mô tả công việc'
end
FtpSever.import_data_from_csv
end
end
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment