Skip to content

V
venjob
Commit b7d32095 by Huynh Thien Phuoc
Options

Created Crawler and Import CSV

parent 363e2ef7
Showing with 346 additions and 59 deletions
.~lock.jobs.csv#
,phuocht,devops-OptiPlex-3010,23.07.2020 08:12,file:///home/phuocht/.config/libreoffice/4; ,phuocht,devops-OptiPlex-3010,27.07.2020 10:41,file:///home/phuocht/.config/libreoffice/4;
\ No newline at end of file \ No newline at end of file
Gemfile
...@@ -27,13 +27,13 @@ gem 'jbuilder', '~> 2.5' ...@@ -27,13 +27,13 @@ gem 'jbuilder', '~> 2.5'
# gem 'redis', '~> 4.0' # gem 'redis', '~> 4.0'
# Use ActiveModel has_secure_password # Use ActiveModel has_secure_password
# gem 'bcrypt', '~> 3.1.7' # gem 'bcrypt', '~> 3.1.7'
gem 'whenever', require: false
# Use ActiveStorage variant # Use ActiveStorage variant
# gem 'mini_magick', '~> 4.8' # gem 'mini_magick', '~> 4.8'
gem 'rubocop', '~> 0.88.0', require: false
# Use Capistrano for deployment # Use Capistrano for deployment
# gem 'capistrano-rails', group: :development # gem 'capistrano-rails', group: :development
gem 'will_paginate', '3.3.0'
# Reduces boot times through caching; required in config/boot.rb # Reduces boot times through caching; required in config/boot.rb
gem 'bootsnap', '>= 1.1.0', require: false gem 'bootsnap', '>= 1.1.0', require: false
......
Gemfile.lock
...@@ -47,6 +47,7 @@ GEM ...@@ -47,6 +47,7 @@ GEM
archive-zip (0.12.0) archive-zip (0.12.0)
io-like (~> 0.3.0) io-like (~> 0.3.0)
arel (9.0.0) arel (9.0.0)
ast (2.4.1)
bindex (0.8.1) bindex (0.8.1)
bootsnap (1.4.6) bootsnap (1.4.6)
msgpack (~> 1.0) msgpack (~> 1.0)
...@@ -64,6 +65,7 @@ GEM ...@@ -64,6 +65,7 @@ GEM
chromedriver-helper (2.1.1) chromedriver-helper (2.1.1)
archive-zip (~> 0.10) archive-zip (~> 0.10)
nokogiri (~> 1.8) nokogiri (~> 1.8)
chronic (0.10.2)
coffee-rails (4.2.2) coffee-rails (4.2.2)
coffee-script (>= 2.2.0) coffee-script (>= 2.2.0)
railties (>= 4.0.0) railties (>= 4.0.0)
...@@ -125,6 +127,9 @@ GEM ...@@ -125,6 +127,9 @@ GEM
nokogiri (1.10.10) nokogiri (1.10.10)
mini_portile2 (~> 2.4.0) mini_portile2 (~> 2.4.0)
ntlm-http (0.1.1) ntlm-http (0.1.1)
parallel (1.19.2)
parser (2.7.1.4)
ast (~> 2.4.1)
public_suffix (4.0.5) public_suffix (4.0.5)
puma (3.12.6) puma (3.12.6)
rack (2.2.3) rack (2.2.3)
...@@ -154,11 +159,25 @@ GEM ...@@ -154,11 +159,25 @@ GEM
method_source method_source
rake (>= 0.8.7) rake (>= 0.8.7)
thor (>= 0.19.0, < 2.0) thor (>= 0.19.0, < 2.0)
rainbow (3.0.0)
rake (13.0.1) rake (13.0.1)
rb-fsevent (0.10.4) rb-fsevent (0.10.4)
rb-inotify (0.10.1) rb-inotify (0.10.1)
ffi (~> 1.0) ffi (~> 1.0)
regexp_parser (1.7.1) regexp_parser (1.7.1)
rexml (3.2.4)
rubocop (0.88.0)
parallel (~> 1.10)
parser (>= 2.7.1.1)
rainbow (>= 2.2.2, < 4.0)
regexp_parser (>= 1.7)
rexml
rubocop-ast (>= 0.1.0, < 1.0)
ruby-progressbar (~> 1.7)
unicode-display_width (>= 1.4.0, < 2.0)
rubocop-ast (0.2.0)
parser (>= 2.7.0.1)
ruby-progressbar (1.10.1)
ruby_dep (1.5.0) ruby_dep (1.5.0)
rubyzip (2.3.0) rubyzip (2.3.0)
sass (3.7.4) sass (3.7.4)
...@@ -199,6 +218,7 @@ GEM ...@@ -199,6 +218,7 @@ GEM
unf (0.1.4) unf (0.1.4)
unf_ext unf_ext
unf_ext (0.0.7.7) unf_ext (0.0.7.7)
unicode-display_width (1.7.0)
web-console (3.7.0) web-console (3.7.0)
actionview (>= 5.0) actionview (>= 5.0)
activemodel (>= 5.0) activemodel (>= 5.0)
...@@ -208,6 +228,9 @@ GEM ...@@ -208,6 +228,9 @@ GEM
websocket-driver (0.7.3) websocket-driver (0.7.3)
websocket-extensions (>= 0.1.0) websocket-extensions (>= 0.1.0)
websocket-extensions (0.1.5) websocket-extensions (0.1.5)
whenever (1.0.0)
chronic (>= 0.6.3)
will_paginate (3.3.0)
xpath (3.2.0) xpath (3.2.0)
nokogiri (~> 1.8) nokogiri (~> 1.8)
...@@ -226,6 +249,7 @@ DEPENDENCIES ...@@ -226,6 +249,7 @@ DEPENDENCIES
mysql2 (~> 0.5.3) mysql2 (~> 0.5.3)
puma (~> 3.11) puma (~> 3.11)
rails (~> 5.2.4, >= 5.2.4.3) rails (~> 5.2.4, >= 5.2.4.3)
rubocop (~> 0.88.0)
sass-rails (~> 5.0) sass-rails (~> 5.0)
selenium-webdriver selenium-webdriver
spring spring
...@@ -234,6 +258,8 @@ DEPENDENCIES ...@@ -234,6 +258,8 @@ DEPENDENCIES
tzinfo-data tzinfo-data
uglifier (>= 1.3.0) uglifier (>= 1.3.0)
web-console (>= 3.3.0) web-console (>= 3.3.0)
whenever
will_paginate (= 3.3.0)
RUBY VERSION RUBY VERSION
ruby 2.6.6p146 ruby 2.6.6p146
......
app/assets/javascripts/top_pages.coffee 0 → 100644
# Place all the behaviors and hooks related to the matching controller here.
# All this logic will automatically be available in application.js.
# You can use CoffeeScript in this file: http://coffeescript.org/
app/assets/stylesheets/custom.css.scss
html { html {
overflow-y: scroll; overflow-y: scroll;
padding: 0px;
margin: 0px;
} }
body {
padding-top: 60px;
}
section { section {
overflow: auto; overflow: auto;
} }
...@@ -48,7 +45,36 @@ p { ...@@ -48,7 +45,36 @@ p {
font-size: 1.1em; font-size: 1.1em;
line-height: 1.7em; line-height: 1.7em;
} }
#logo{ .footer-text{
height: 200px; height: 100px;
width: 100px; position: relative;
}
.float-right {
position: absolute;
bottom: 0;
right: 5%;
}
.top-page{
text-align: center;
width: 100%;
height: 100%;
}
.total-job{
position: absolute;
color: white;
top: 40%;
right: 30%;
font-size: 60px;
}
.banner-ground{
position: relative;
width: 100%;
height: 50vh;
} }
.top-banner {
background-image: url('banner.jpg');
width: 100%;
height: 100%;
background-repeat: no-repeat;
background-size: cover;
}
\ No newline at end of file
app/assets/stylesheets/top_pages.scss 0 → 100644
// Place all the styles related to the Top_pages controller here.
// They will automatically be included in application.css.
// You can use Sass (SCSS) here: http://sass-lang.com/
app/controllers/jobs_controller.rb
class JobsController < ApplicationController class JobsController < ApplicationController
def index def index
end end
end end
app/controllers/top_pages_controller.rb 0 → 100644
class TopPagesController < ApplicationController
def index
@total = Job.ids
end
def show
@job = Job.all
@company = Company.all
end
end
app/helpers/application_helper.rb
module ApplicationHelper module ApplicationHelper
def full_title(page_title)
"#{page_title}"
end
end end
app/helpers/top_pages_helper.rb 0 → 100644
module TopPagesHelper
end
app/models/top_page.rb 0 → 100644
class TopPage < ApplicationRecord
# scope :limit_job, -> { order('created_at DESC') }
scope :limit_job, -> { order('title DESC') }
# scope :limit_job, order(created_at: :desc).limit(5)
end
app/views/layouts/_footer.html.erb 0 → 100644
<div class="p-3 mb-2 bg-dark text-white">
<div class="footer-text">
<div class="float-right">
Copyright © ZIGExN VeNtura 2020
</div>
</div>
</div>
\ No newline at end of file
app/views/layouts/_header.html.erb 0 → 100644
<header class="navbar navbar-expand-lg navbar-dark bg-dark">
<div class="collapse navbar-collapse">
<div class="container-fluid">
<%= link_to image_tag("logo_venjob.png", alt: "Logo", id: "logo_venjob", width: 120), root_path %>
<nav>
<ul class="navbar-nav mr-auto">
<li><%= link_to "Login", '#' , class: "nav-item nav-link" %></li>
<li><%= link_to "Register", '#', class: "nav-item nav-link" %></li>
<li><%= link_to "Favorite", '#', class: "nav-item nav-link" %></li>
<li><%= link_to "History", '#', class: "nav-item nav-link" %></li>
</ul>
</nav>
</div>
</div>
</header>
\ No newline at end of file
app/views/layouts/application.html.erb
<!DOCTYPE html> <!DOCTYPE html>
<html> <html>
<head> <head>
<title>Venjob</title> <title><%= full_title(yield(:title)) %></title>
<%= csrf_meta_tags %> <%= csrf_meta_tags %>
<%= csp_meta_tag %> <%= csp_meta_tag %>
...@@ -10,25 +10,13 @@ ...@@ -10,25 +10,13 @@
<%= stylesheet_link_tag 'bootstrap', media: 'all', 'data-turbolinks-track': 'reload' %> <%= stylesheet_link_tag 'bootstrap', media: 'all', 'data-turbolinks-track': 'reload' %>
<%= javascript_include_tag 'bootstrap', media: 'all', 'data-turbolinks-track': 'reload' %> <%= javascript_include_tag 'bootstrap', media: 'all', 'data-turbolinks-track': 'reload' %>
<%= javascript_include_tag 'https://kit.fontawesome.com/yourcode.js', media: 'all', 'data-turbolinks-track': 'reload' %>
</head> </head>
<body> <body>
<header class="navbar navbar-expand-lg navbar-dark bg-dark"> <%= render "layouts/header" %>
<div class="collapse navbar-collapse">
<div class="container">
<%= link_to image_tag("logo_venjob.png", alt: "Logo", id: "logo_venjob", width: 120, left: 0), '#'%>
<nav>
<ul class="navbar-nav mr-auto">
<li><%= link_to "Login", '#' %></li>
<li><%= link_to "Register", '#' %></li>
<li><%= link_to "Favorite", '#' %></li>
<li><%= link_to "History", '#' %></li>
</ul>
</nav>
</div>
</div>
</header>
<div class="container">
<%= yield %> <%= yield %>
</div>
<%= render "layouts/footer" %>
</body> </body>
</html> </html>
app/views/top_pages/index.html.erb 0 → 100644
<% provide(:title, 'Venjob') %>
<div class="banner-ground">
<div class="top-banner">
<div class="total-job">total: <%= @total.count %> Jobs</div>
</div>
</div>
<div><%#= render 'top_pages/show' %></div>
app/views/top_pages/show.html.erb 0 → 100644
<% provide(:title, 'All jobs') %>
<% @job.each do |job| %>
<ul>
<div class="title"><strong><%= (@company.find_by(id: job.company_id)).name %></strong></div>
<%= job.title %>
<div class="salary"><em class="fa fa-usd"></em>Lương: <%= job.salary %></div>
<div><em class="mdi mdi-map-marker"></em>
<% job.cities.each do |location| %>
<%= location.name %>
<% end %>
</div>
</ul>
<% end %>
\ No newline at end of file
config/database.yml
...@@ -26,6 +26,6 @@ test: ...@@ -26,6 +26,6 @@ test:
production: production:
<<: *default <<: *default
database: buildvenjob database: phuociadun
username: root username: root
password: 123456 password: 123456
\ No newline at end of file
config/routes.rb
Rails.application.routes.draw do Rails.application.routes.draw do
# For details on the DSL available within this file, see http://guides.rubyonrails.org/routing.html # For details on the DSL available within this file, see http://guides.rubyonrails.org/routing.html
resources :jobs resources :jobs
resources :top_pages
root to: "top_pages#index"
end end
config/schedule.rb 0 → 100644
env :PATH, ENV['PATH']
# every 10.minutes do
# rake 'import:auto'
# end
db/migrate/20200723071239_create_top_pages.rb 0 → 100644
class CreateTopPages < ActiveRecord::Migration[5.2]
def change
create_table :top_pages do |t|
t.timestamps
end
end
end
db/schema.rb
...@@ -10,7 +10,7 @@ ...@@ -10,7 +10,7 @@
# #
# It's strongly recommended that you check this file into your version control system. # It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema.define(version: 2020_07_15_035356) do ActiveRecord::Schema.define(version: 2020_07_23_071239) do
create_table "cities", options: "ENGINE=InnoDB DEFAULT CHARSET=utf8", force: :cascade do |t| create_table "cities", options: "ENGINE=InnoDB DEFAULT CHARSET=utf8", force: :cascade do |t|
t.string "name" t.string "name"
...@@ -83,6 +83,11 @@ ActiveRecord::Schema.define(version: 2020_07_15_035356) do ...@@ -83,6 +83,11 @@ ActiveRecord::Schema.define(version: 2020_07_15_035356) do
t.datetime "updated_at", null: false t.datetime "updated_at", null: false
end end
create_table "top_pages", options: "ENGINE=InnoDB DEFAULT CHARSET=utf8", force: :cascade do |t|
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
end
create_table "users", options: "ENGINE=InnoDB DEFAULT CHARSET=utf8", force: :cascade do |t| create_table "users", options: "ENGINE=InnoDB DEFAULT CHARSET=utf8", force: :cascade do |t|
t.string "name" t.string "name"
t.string "email" t.string "email"
......
jobs.csv
This source diff could not be displayed because it is too large. You can view the blob instead.
lib/src/crawler.rb
...@@ -75,7 +75,7 @@ ...@@ -75,7 +75,7 @@
if val.include?('Ngày cập nhật') if val.include?('Ngày cập nhật')
arr_data = val.gsub('Ngày cập nhật ','').split(' ') arr_data = val.gsub('Ngày cập nhật ','').split(' ')
date = arr_data.first date = arr_data.first
elsif val.include?('Lương') && val.include?('Kinh nghiệm') == true elsif val.include?('Lương') && val.include?('Kinh nghiệm') == true && Job.find_by(title: title_job, company_id: company_table.id) == nil
arr_sub = ((((val.gsub('Lương ','')).gsub(' Kinh nghiệm ', '*')).gsub(' Cấp bậc ', '*')).gsub(' Hết hạn nộp ', '*')).split('*') arr_sub = ((((val.gsub('Lương ','')).gsub(' Kinh nghiệm ', '*')).gsub(' Cấp bậc ', '*')).gsub(' Hết hạn nộp ', '*')).split('*')
salary = arr_sub[0] salary = arr_sub[0]
experience = arr_sub[1] experience = arr_sub[1]
...@@ -88,7 +88,7 @@ ...@@ -88,7 +88,7 @@
expiration_date: expiration_date, expiration_date: expiration_date,
description: description, description: description,
company_id: company_table.id) company_id: company_table.id)
elsif val.include?('Lương') && val.include?('Kinh nghiệm') == false elsif val.include?('Lương') && val.include?('Kinh nghiệm') == false && Job.find_by(title: title_job, company_id: company_table.id) == nil
arr_sub = (((val.gsub('Lương ','')).gsub(' Cấp bậc ', '*')).gsub(' Hết hạn nộp ', '*')).split('*') arr_sub = (((val.gsub('Lương ','')).gsub(' Cấp bậc ', '*')).gsub(' Hết hạn nộp ', '*')).split('*')
salary = arr_sub[0] salary = arr_sub[0]
level = arr_sub[1] level = arr_sub[1]
...@@ -102,23 +102,23 @@ ...@@ -102,23 +102,23 @@
company_id: company_table.id) company_id: company_table.id)
end end
end end
end job_table = Job.find_by(title: title_job)
job_table = Job.find_by(title: "#{title_job}") if job_table != nil
if job_table != nil
location_rel = get_row.css('div.map p a').children.map{ |location| location.text.strip } location_rel = get_row.css('div.map p a').children.map{ |location| location.text.strip }
location_rel.each do |loc| location_rel.each do |loc|
puts "#{job_table.id} - #{loc}" city_table = City.find_by(name: "#{loc}")
city_table = City.find_by(name: "#{loc}") puts "Created City: #{job_table.id} - #{city_table.id}.#{loc}"
city_jobs = CityJob.create!(job_id: job_table.id, city_id: city_table.id) city_jobs = CityJob.create!(job_id: job_table.id, city_id: city_table.id)
end end
industry_rel = get_row.css('li a').children.map{ |industry| industry.text.strip } industry_rel = get_row.css('li a').children.map{ |industry| industry.text.strip }
industry_rel.each do |ind| industry_rel.each do |ind|
puts "#{job_table.id} - #{ind}" industry_table = Industry.find_by(name: "#{ind}")
industry_table = Industry.find_by(name: "#{ind}") puts "Created Industry: #{job_table.id} - #{industry_table.id}.#{ind}"
industry_jobs = IndustryJob.create!(job_id: job_table.id, industry_id: industry_table.id) industry_jobs = IndustryJob.create!(job_id: job_table.id, industry_id: industry_table.id)
end end
end end
end end
end
end end
end end
end end
...@@ -126,7 +126,7 @@ ...@@ -126,7 +126,7 @@
def get_file_csv def get_file_csv
Net::FTP.open('192.168.1.156', 'training', 'training') do |ftp| Net::FTP.open('192.168.1.156', 'training', 'training') do |ftp|
files = ftp.list files = ftp.list
puts "list out files in root directory:" puts "list files:"
puts files puts files
ftp.getbinaryfile('jobs.zip') ftp.getbinaryfile('jobs.zip')
end end
...@@ -143,50 +143,59 @@ ...@@ -143,50 +143,59 @@
end end
def import_file_csv def import_file_csv
file = "jobs.csv" file = "jobs.csv"
CSV.foreach(file, headers: true) do |row| CSV.foreach(file, headers: true) do |row|
begin begin
company_name = row[5].strip company_name = row["company name"].strip
company_address = row[2] company_address = row["company address"]
company_introduction = row[0] company_introduction = row["benefit"]
company_table = Company.find_by(name: "#{company_name}") company_table = Company.find_by(name: "#{company_name}")
if company_table == nil if company_table == nil
company_table = Company.create!(name: company_name, company_table = Company.create!(name: company_name,
address: company_address, address: company_address,
introduction: company_introduction) introduction: company_introduction)
end end
title_job = row[9].strip title_job = row["name"].strip
description_job = row[7] description_job = row["description"]
level = row[8] level = row["level"]
salary = row[11] salary = row["salary"]
if company_table != nil if company_table != nil && Job.find_by(title: title_job, level: level, salary: salary, company_id: company_table.id) == nil
job_table = Job.create!(title: title_job, job_table = Job.create!(title: title_job,
description: description_job, description: description_job,
level: level, level: level,
salary: salary, salary: salary,
company_id: company_table.id) company_id: company_table.id)
end end
industry = row[1].strip industry = row["category"].strip
industry_find = Industry.find_by(name: industry) industry_find = Industry.find_by(name: industry)
if industry_find == nil if industry_find == nil
industry_table = Industry.create!(name: industry) industry_table = Industry.create!(name: industry)
industry_job_table = IndustryJob.create!(job_id: job_table.id, industry_id: industry_find.id) industry_job_table = IndustryJob.create!(job_id: job_table.id, industry_id: industry_find.id)
elsif industry_find != nil else
industry_job_table = IndustryJob.create!(job_id: job_table.id, industry_id: industry_find.id) industry_job_table = IndustryJob.create!(job_id: job_table.id, industry_id: industry_find.id)
end end
puts "=========================================" puts "========================================="
puts job_table.id, title_job, industry, salary puts job_table.id, title_job, industry, salary
location_data = row[16].strip location_data = row["work place"].strip
location = (location_data.gsub('["','')).gsub('"]','').strip location = (location_data.gsub('["','')).gsub('"]','').strip
location_find = City.find_by(name: location) location_find = City.find_by(name: location)
if location_find != nil if location_find == nil
city_table = City.create!(name: location)
city_job_table = CityJob.create!(job_id: job_table.id, city_id: location_find.id)
else
city_job_table = CityJob.create!(job_id: job_table.id, city_id: location_find.id) city_job_table = CityJob.create!(job_id: job_table.id, city_id: location_find.id)
end end
puts "Location: #{location}" puts "Location: #{location}"
rescue StandardError => e rescue StandardError => e
puts e puts e
end end
end end
end end
def logger
# config.log_level = :info
Rails.logger = Logger.new(STDOUT)
Rails.logger = Logger.new "#{Rails.root}/log/#{Rails.env}.log"
Rails.logger.level = Logger::DEBUG
Rails.logger.datetime_format = "%Y-%m-%d %H:%M:%S"
end
end end
\ No newline at end of file
lib/src/crontab.rb 0 → 100644
class Crontab
def find_company
company_info = Nokogiri::HTML(URI.open("https://careerbuilder.vn/viec-lam/tat-ca-viec-lam-trang-1-vi.html"))
company_link = company_info.css('div.caption a.company-name').map{ |link| link['href'] }
company_link.each do |link|
if link.include?('\u2019')
link.gsub!('\u2019',"'")
end
if link == 'javascript:void(0);'
next
elsif link != 'https://careerbuilder.vn/vi/nha-tuyen-dung/hr-vietnam\xE2\x80\x99s-ess-client.35A4EFBA.html'
company_page = Nokogiri::HTML(URI.open(URI.parse(URI.escape(link))))
if company_page.search('p.name').text != nil
begin
name_company = company_page.search('p.name').text
address_company = company_page.css('div.content p').children[1].text
introduction_company = company_page.css('div.main-about-us').text
get_name_company = Company.find_by(name: "#{name_company}")
if get_name_company == nil
company = Company.create!(name: name_company,
address: address_company,
introduction: introduction_company)
end
rescue StandardError => e
puts e
end
end
end
end
end
def find_job
page_access = Nokogiri::HTML(URI.open("https://careerbuilder.vn/viec-lam/tat-ca-viec-lam-vi.html"))
get_link = page_access.css('a.job_link').map{ |link| link['href'] }
get_link.each do |link|
if link.include?('\u2013')
link.gsub!('\u2013','–')
end
page_job = Nokogiri::HTML(URI.open(URI.parse(URI.escape(link))))
get_row = page_job.search('div.bg-blue div.row')
if get_row != ""
get_name_company = page_job.search('div.job-desc a.job-company-name').text.strip
company_table = Company.find_by(name: "#{get_name_company}")
title_job = page_job.search('div.job-desc p').text
description = page_job.search('div.detail-row')
arr_column = get_row.css('div.has-background').map{ |data| data.text.split(' ').join(' ') }
job_table = Job.find_by(title: "#{title_job}")
arr_column.each_with_index do | val, key |
if company_table != nil
if val.include?('Ngày cập nhật')
arr_data = val.gsub('Ngày cập nhật ','').split(' ')
date = arr_data.first
elsif val.include?('Lương') && val.include?('Kinh nghiệm') == true && Job.find_by(title: title_job, company_id: company_table.id) == nil
arr_sub = ((((val.gsub('Lương ','')).gsub(' Kinh nghiệm ', '*')).gsub(' Cấp bậc ', '*')).gsub(' Hết hạn nộp ', '*')).split('*')
salary = arr_sub[0]
experience = arr_sub[1]
level = arr_sub[2]
expiration_date = arr_sub[3]
job = Job.create!(title: title_job,
level: level,
salary: salary,
experience: experience,
expiration_date: expiration_date,
description: description,
company_id: company_table.id)
elsif val.include?('Lương') && val.include?('Kinh nghiệm') == false && Job.find_by(title: title_job, company_id: company_table.id) == nil
arr_sub = (((val.gsub('Lương ','')).gsub(' Cấp bậc ', '*')).gsub(' Hết hạn nộp ', '*')).split('*')
salary = arr_sub[0]
level = arr_sub[1]
expiration_date = arr_sub[2]
job = Job.create!(title: title_job,
level: level,
salary: salary,
experience: 'Không có',
expiration_date: expiration_date,
description: description,
company_id: company_table.id)
end
end
end
if !job_table.nil? && !company_table.nil?
location_rel = get_row.css('div.map p a').children.map{ |location| location.text.strip }
location_rel.each do |loc|
city_table = City.find_by(name: "#{loc}")
if CityJob.find_by(job_id: job_table.id, city_id: city_table.id) == nil
puts "Created #{job_table.id} - #{city_table.id}.#{loc}"
city_jobs = CityJob.create!(job_id: job_table.id, city_id: city_table.id)
end
end
industry_rel = get_row.css('li a').children.map{ |industry| industry.text.strip }
industry_rel.each do |ind|
industry_table = Industry.find_by(name: "#{ind}")
if IndustryJob.find_by(job_id: job_table.id, industry_id: industry_table.id) == nil
puts "#{job_table.id} - #{industry_table.id}.#{ind}"
industry_jobs = IndustryJob.create!(job_id: job_table.id, industry_id: industry_table.id)
end
end
end
end
end
end
end
\ No newline at end of file
lib/tasks/crawler_import.rake
require 'src/crawler.rb' require 'src/crawler.rb'
require 'src/crontab.rb'
require 'net/ftp' require 'net/ftp'
require 'csv' require 'csv'
require 'zip' require 'zip'
action = Crawler.new action = Crawler.new
crontab = Crontab.new
namespace :import do namespace :import do
desc "crawler data" desc "crawler data"
task crawler: :environment do task crawler: :environment do
...@@ -20,4 +22,15 @@ namespace :import do ...@@ -20,4 +22,15 @@ namespace :import do
task data_csv: :environment do task data_csv: :environment do
action.import_file_csv action.import_file_csv
end end
desc "Crontab"
task auto: :environment do
crontab.find_company
crontab.find_job
action.logger
end
task log: :environment do
action.logger
end
end end
\ No newline at end of file
logfile.log 0 → 100644
# Logfile created on 2020-07-23 16:15:50 +0700 by logger.rb/66358
test/controllers/top_pages_controller_test.rb 0 → 100644
require 'test_helper'
class TopPagesControllerTest < ActionDispatch::IntegrationTest
# test "the truth" do
# assert true
# end
end
test/fixtures/top_pages.yml 0 → 100644
# Read about fixtures at http://api.rubyonrails.org/classes/ActiveRecord/FixtureSet.html
# This model initially had no columns defined. If you add columns to the
# model remove the '{}' from the fixture names and add the columns immediately
# below each fixture, per the syntax in the comments below
#
one: {}
# column: value
#
two: {}
# column: value
test/models/top_page_test.rb 0 → 100644
require 'test_helper'
class TopPageTest < ActiveSupport::TestCase
# test "the truth" do
# assert true
# end
end
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment