Skip to content

V
VenJob
Commit 2b3e5d23 by Mai Hoang Thai Ha
Options

add argument to crawl 5 or all page ,merge branch migration, import data to City and Industry

parent 6aa95db8
Showing with 156 additions and 5 deletions
fasralsc 0 → 100644
=> [#<Industry:0x00005603efca8a48 id: 1, name: "Tiếp thị / Marketing", created_at: Wed, 14 Jul 2021 08:45:42.708737000 UTC +00:00, updated_at: Wed, 14 Jul 2021 08:45:42.708737000 UTC +00:00>,
#<Industry:0x00005603efd793c8 id: 2, name: "Tiếp thị trực tuyến", created_at: Wed, 14 Jul 2021 08:45:42.720561000 UTC +00:00, updated_at: Wed, 14 Jul 2021 08:45:42.720561000 UTC +00:00>,
#<Industry:0x00005603efd791c0 id: 3, name: "Bán lẻ / Bán sỉ", created_at: Wed, 14 Jul 2021 08:45:42.731511000 UTC +00:00, updated_at: Wed, 14 Jul 2021 08:45:42.731511000 UTC +00:00>,
#<Industry:0x00005603efd78fe0 id: 4, name: "Bán hàng / Kinh doanh", created_at: Wed, 14 Jul 2021 08:45:42.740572000 UTC +00:00, updated_at: Wed, 14 Jul 2021 08:45:42.740572000 UTC +00:00>,
#<Industry:0x00005603efd789c8 id: 5, name: "Tư vấn", created_at: Wed, 14 Jul 2021 08:45:42.749196000 UTC +00:00, updated_at: Wed, 14 Jul 2021 08:45:42.749196000 UTC +00:00>,
#<Industry:0x00005603efd78248 id: 6, name: "An Ninh / Bảo Vệ", created_at: Wed, 14 Jul 2021 08:45:42.757834000 UTC +00:00, updated_at: Wed, 14 Jul 2021 08:45:42.757834000 UTC +00:00>,
#<Industry:0x00005603efd73f18 id: 7, name: "Luật / Pháp lý", created_at: Wed, 14 Jul 2021 08:45:42.769893000 UTC +00:00, updated_at: Wed, 14 Jul 2021 08:45:42.769893000 UTC +00:00>,
#<Industry:0x00005603efd73d60 id: 8, name: "Dịch vụ khách hàng", created_at: Wed, 14 Jul 2021 08:45:42.779790000 UTC +00:00, updated_at: Wed, 14 Jul 2021 08:45:42.779790000 UTC +00:00>,
#<Industry:0x00005603efd73ba8 id: 9, name: "Lao động phổ thông", created_at: Wed, 14 Jul 2021 08:45:42.788162000 UTC +00:00, updated_at: Wed, 14 Jul 2021 08:45:42.788162000 UTC +00:00>,
#<Industry:0x00005603efd73658 id: 10, name: "Bưu chính viễn thông", created_at: Wed, 14 Jul 2021 08:45:42.795903000 UTC +00:00, updated_at: Wed, 14 Jul 2021 08:45:42.795903000 UTC +00:00>,
#<Industry:0x00005603efd732c0
id: 11,
name: "Vận chuyển / Giao nhận / Kho vận",
created_at: Wed, 14 Jul 2021 08:45:42.803702000 UTC +00:00,
updated_at: Wed, 14 Jul 2021 08:45:42.803702000 UTC +00:00>,
#<Industry:0x00005603efd73158
id: 12,
name: "Phi chính phủ / Phi lợi nhuận",
created_at: Wed, 14 Jul 2021 08:45:42.811514000 UTC +00:00,
updated_at: Wed, 14 Jul 2021 08:45:42.811514000 UTC +00:00>,
#<Industry:0x00005603efd72f50
id: 13,
name: "Y tế / Chăm sóc sức khỏe",
created_at: Wed, 14 Jul 2021 08:45:42.821276000 UTC +00:00,
updated_at: Wed, 14 Jul 2021 08:45:42.821276000 UTC +00:00>,
#<Industry:0x00005603efd72e38 id: 14, name: "Dược phẩm", created_at: Wed, 14 Jul 2021 08:45:42.828775000 UTC +00:00, updated_at: Wed, 14 Jul 2021 08:45:42.828775000 UTC +00:00>,
#<Industry:0x00005603efd72be0
id: 15,
name: "Dệt may / Da giày / Thời trang",
created_at: Wed, 14 Jul 2021 08:45:42.836578000 UTC +00:00,
updated_at: Wed, 14 Jul 2021 08:45:42.836578000 UTC +00:00>,
#<Industry:0x00005603efd728c0
id: 16,
name: "Quản lý chất lượng (QA/QC)",
created_at: Wed, 14 Jul 2021 08:45:42.844295000 UTC +00:00,
updated_at: Wed, 14 Jul 2021 08:45:42.844295000 UTC +00:00>,
#<Industry:0x00005603efd72730 id: 17, name: "Thu mua / Vật tư", created_at: Wed, 14 Jul 2021 08:45:42.852071000 UTC +00:00, updated_at: Wed, 14 Jul 2021 08:45:42.852071000 UTC +00:00>,
#<Industry:0x00005603efd71d30 id: 18, name: "An toàn lao động", created_at: Wed, 14 Jul 2021 08:45:42.860094000 UTC +00:00, updated_at: Wed, 14 Jul 2021 08:45:42.860094000 UTC +00:00>,
#<Industry:0x00005603efd71920
id: 19,
name: "Sản xuất / Vận hành sản xuất",
created_at: Wed, 14 Jul 2021 08:45:42.869735000 UTC +00:00,
updated_at: Wed, 14 Jul 2021 08:45:42.869735000 UTC +00:00>,
#<Industry:0x00005603efd71178 id: 20, name: "Xuất nhập khẩu", created_at: Wed, 14 Jul 2021 08:45:42.880983000 UTC +00:00, updated_at: Wed, 14 Jul 2021 08:45:42.880983000 UTC +00:00>,
#<Industry:0x00005603efd71088 id: 21, name: "Đồ gỗ", created_at: Wed, 14 Jul 2021 08:45:42.903977000 UTC +00:00, updated_at: Wed, 14 Jul 2021 08:45:42.903977000 UTC +00:00>,
#<Industry:0x00005603efd70f48 id: 22, name: "In ấn / Xuất bản", created_at: Wed, 14 Jul 2021 08:45:42.913203000 UTC +00:00, updated_at: Wed, 14 Jul 2021 08:45:42.913203000 UTC +00:00>,
#<Industry:0x00005603efd70e58
id: 23,
name: "Hàng gia dụng / Chăm sóc cá nhân",
created_at: Wed, 14 Jul 2021 08:45:42.922009000 UTC +00:00,
updated_at: Wed, 14 Jul 2021 08:45:42.922009000 UTC +00:00>,
#<Industry:0x00005603efd70d40 id: 24, name: "Thực phẩm & Đồ uống", created_at: Wed, 14 Jul 2021 08:45:42.931785000 UTC +00:00, updated_at: Wed, 14 Jul 2021 08:45:42.931785000 UTC +00:00>,
#<Industry:0x00005603efd70b60 id: 25, name: "CNTT - Phần mềm", created_at: Wed, 14 Jul 2021 08:45:42.940991000 UTC +00:00, updated_at: Wed, 14 Jul 2021 08:45:42.940991000 UTC +00:00>,
#<Industry:0x00005603efd708e0
id: 26,
name: "CNTT - Phần cứng / Mạng",
created_at: Wed, 14 Jul 2021 08:45:42.973373000 UTC +00:00,
updated_at: Wed, 14 Jul 2021 08:45:42.973373000 UTC +00:00>,
#<Industry:0x00005603efd70700 id: 27, name: "Nhân sự", created_at: Wed, 14 Jul 2021 08:45:42.987748000 UTC +00:00, updated_at: Wed, 14 Jul 2021 08:45:42.987748000 UTC +00:00>,
#<Industry:0x00005603efd70480 id: 28, name: "Hành chính / Thư ký", created_at: Wed, 14 Jul 2021 08:45:43.002179000 UTC +00:00, updated_at: Wed, 14 Jul 2021 08:45:43.002179000 UTC +00:00>,
#<Industry:0x00005603efd6bcf0 id: 29, name: "Biên phiên dịch", created_at: Wed, 14 Jul 2021 08:45:43.016101000 UTC +00:00, updated_at: Wed, 14 Jul 2021 08:45:43.016101000 UTC +00:00>,
#<Industry:0x00005603efd6ba70 id: 30, name: "Ngân hàng", created_at: Wed, 14 Jul 2021 08:45:43.025700000 UTC +00:00, updated_at: Wed, 14 Jul 2021 08:45:43.025700000 UTC +00:00>,
#<Industry:0x00005603efd6b908 id: 31, name: "Chứng khoán", created_at: Wed, 14 Jul 2021 08:45:43.038132000 UTC +00:00, updated_at: Wed, 14 Jul 2021 08:45:43.038132000 UTC +00:00>,
#<Industry:0x00005603efd6b6d8 id: 32, name: "Bảo hiểm", created_at: Wed, 14 Jul 2021 08:45:43.052405000 UTC +00:00, updated_at: Wed, 14 Jul 2021 08:45:43.052405000 UTC +00:00>,
#<Industry:0x00005603efd6b4a8 id: 33, name: "Tài chính / Đầu tư", created_at: Wed, 14 Jul 2021 08:45:43.066092000 UTC +00:00, updated_at: Wed, 14 Jul 2021 08:45:43.066092000 UTC +00:00>,
#<Industry:0x00005603efd6b2f0 id: 34, name: "Kế toán / Kiểm toán", created_at: Wed, 14 Jul 2021 08:45:43.075569000 UTC +00:00, updated_at: Wed, 14 Jul 2021 08:45:43.075569000 UTC +00:00>,
#<Industry:0x00005603efd6b200 id: 35, name: "Tổ chức sự kiện", created_at: Wed, 14 Jul 2021 08:45:43.084159000 UTC +00:00, updated_at: Wed, 14 Jul 2021 08:45:43.084159000 UTC +00:00>,
#<Industry:0x00005603efd6ad50
id: 36,
name: "Truyền hình / Báo chí / Biên tập",
created_at: Wed, 14 Jul 2021 08:45:43.093434000 UTC +00:00,
updated_at: Wed, 14 Jul 2021 08:45:43.093434000 UTC +00:00>,
#<Industry:0x00005603efd6a9e0
id: 37,
name: "Mỹ thuật / Nghệ thuật / Thiết kế",
created_at: Wed, 14 Jul 2021 08:45:43.104974000 UTC +00:00,
updated_at: Wed, 14 Jul 2021 08:45:43.104974000 UTC +00:00>,
#<Industry:0x00005603efd6a710
id: 38,
name: "Quảng cáo / Đối ngoại / Truyền Thông",
created_at: Wed, 14 Jul 2021 08:45:43.122039000 UTC +00:00,
updated_at: Wed, 14 Jul 2021 08:45:43.122039000 UTC +00:00>,
#<Industry:0x00005603efd6a328 id: 39, name: "Giải trí", created_at: Wed, 14 Jul 2021 08:45:43.139269000 UTC +00:00, updated_at: Wed, 14 Jul 2021 08:45:43.139269000 UTC +00:00>,
#<Industry:0x00005603efd6a058 id: 40, name: "Bất động sản", created_at: Wed, 14 Jul 2021 08:45:43.157675000 UTC +00:00, updated_at: Wed, 14 Jul 2021 08:45:43.157675000 UTC +00:00>,
#<Industry:0x00005603efd69c98 id: 41, name: "Nội ngoại thất", created_at: Wed, 14 Jul 2021 08:45:43.174550000 UTC +00:00, updated_at: Wed, 14 Jul 2021 08:45:43.174550000 UTC +00:00>,
#<Industry:0x00005603efd695b8 id: 42, name: "Kiến trúc", created_at: Wed, 14 Jul 2021 08:45:43.192179000 UTC +00:00, updated_at: Wed, 14 Jul 2021 08:45:43.192179000 UTC +00:00>,
#<Industry:0x00005603efd69388 id: 43, name: "Xây dựng", created_at: Wed, 14 Jul 2021 08:45:43.206031000 UTC +00:00, updated_at: Wed, 14 Jul 2021 08:45:43.206031000 UTC +00:00>,
#<Industry:0x00005603efd69270
id: 44,
name: "Cơ khí / Ô tô / Tự động hóa",
created_at: Wed, 14 Jul 2021 08:45:43.217299000 UTC +00:00,
updated_at: Wed, 14 Jul 2021 08:45:43.217299000 UTC +00:00>,
#<Industry:0x00005603efd68ff0 id: 45, name: "Môi trường", created_at: Wed, 14 Jul 2021 08:45:43.229813000 UTC +00:00, updated_at: Wed, 14 Jul 2021 08:45:43.229813000 UTC +00:00>,
#<Industry:0x00005603efd68d70
id: 46,
name: "Điện / Điện tử / Điện lạnh",
created_at: Wed, 14 Jul 2021 08:45:43.276779000 UTC +00:00,
updated_at: Wed, 14 Jul 2021 08:45:43.276779000 UTC +00:00>,
#<Industry:0x00005603efd68af0 id: 47, name: "Hóa học", created_at: Wed, 14 Jul 2021 08:45:43.299583000 UTC +00:00, updated_at: Wed, 14 Jul 2021 08:45:43.299583000 UTC +00:00>,
#<Industry:0x00005603efd68898 id: 48, name: "Khoáng sản", created_at: Wed, 14 Jul 2021 08:45:43.336894000 UTC +00:00, updated_at: Wed, 14 Jul 2021 08:45:43.336894000 UTC +00:00>,
#<Industry:0x00005603efd68168 id: 49, name: "Dầu khí", created_at: Wed, 14 Jul 2021 08:45:43.348229000 UTC +00:00, updated_at: Wed, 14 Jul 2021 08:45:43.348229000 UTC +00:00>,
#<Industry:0x00005603efd63fa0 id: 50, name: "Bảo trì / Sửa chữa", created_at: Wed, 14 Jul 2021 08:45:43.361360000 UTC +00:00, updated_at: Wed, 14 Jul 2021 08:45:43.361360000 UTC +00:00>,
#<Industry:0x00005603efd63758 id: 51, name: "Thư viện", created_at: Wed, 14 Jul 2021 08:45:43.377690000 UTC +00:00, updated_at: Wed, 14 Jul 2021 08:45:43.377690000 UTC +00:00>,
#<Industry:0x00005603efd63528 id: 52, name: "Giáo dục / Đào tạo", created_at: Wed, 14 Jul 2021 08:45:43.392635000 UTC +00:00, updated_at: Wed, 14 Jul 2021 08:45:43.392635000 UTC +00:00>,
#<Industry:0x00005603efd63208 id: 53, name: "Lâm Nghiệp", created_at: Wed, 14 Jul 2021 08:45:43.407040000 UTC +00:00, updated_at: Wed, 14 Jul 2021 08:45:43.407040000 UTC +00:00>,
#<Industry:0x00005603efd62e20 id: 54, name: "Thủy sản / Hải sản", created_at: Wed, 14 Jul 2021 08:45:43.416781000 UTC +00:00, updated_at: Wed, 14 Jul 2021 08:45:43.416781000 UTC +00:00>,
#<Industry:0x00005603efd62740
id: 55,
name: "Công nghệ thực phẩm / Dinh dưỡng",
created_at: Wed, 14 Jul 2021 08:45:43.429422000 UTC +00:00,
updated_at: Wed, 14 Jul 2021 08:45:43.429422000 UTC +00:00>,
#<Industry:0x00005603efd624c0 id: 56, name: "Thống kê", created_at: Wed, 14 Jul 2021 08:45:43.440188000 UTC +00:00, updated_at: Wed, 14 Jul 2021 08:45:43.440188000 UTC +00:00>,
#<Industry:0x00005603efd60c60 id: 57, name: "Nông nghiệp", created_at: Wed, 14 Jul 2021 08:45:43.449865000 UTC +00:00, updated_at: Wed, 14 Jul 2021 08:45:43.449865000 UTC +00:00>,
#<Industry:0x00005603efd601e8 id: 58, name: "Hàng hải", created_at: Wed, 14 Jul 2021 08:45:43.459926000 UTC +00:00, updated_at: Wed, 14 Jul 2021 08:45:43.459926000 UTC +00:00>,
#<Industry:0x00005603efd5b170 id: 59, name: "Công nghệ sinh học", created_at: Wed, 14 Jul 2021 08:45:43.471282000 UTC +00:00, updated_at: Wed, 14 Jul 2021 08:45:43.471282000 UTC +00:00>,
#<Industry:0x00005603efd5b058 id: 60, name: "Trắc địa / Địa Chất", created_at: Wed, 14 Jul 2021 08:45:43.480883000 UTC +00:00, updated_at: Wed, 14 Jul 2021 08:45:43.480883000 UTC +00:00>,
#<Industry:0x00005603efd5ae50 id: 61, name: "Thủy lợi", created_at: Wed, 14 Jul 2021 08:45:43.492807000 UTC +00:00, updated_at: Wed, 14 Jul 2021 08:45:43.492807000 UTC +00:00>,
#<Industry:0x00005603efd5ab80 id: 62, name: "Chăn nuôi / Thú y", created_at: Wed, 14 Jul 2021 08:45:43.505134000 UTC +00:00, updated_at: Wed, 14 Jul 2021 08:45:43.505134000 UTC +00:00>,
#<Industry:0x00005603efd5aa40 id: 63, name: "Du lịch", created_at: Wed, 14 Jul 2021 08:45:43.515473000 UTC +00:00, updated_at: Wed, 14 Jul 2021 08:45:43.515473000 UTC +00:00>,
#<Industry:0x00005603efd5a810 id: 64, name: "Nhà hàng / Khách sạn", created_at: Wed, 14 Jul 2021 08:45:43.523137000 UTC +00:00, updated_at: Wed, 14 Jul 2021 08:45:43.523137000 UTC +00:00>,
#<Industry:0x00005603efd5a5b8 id: 65, name: "Hàng không", created_at: Wed, 14 Jul 2021 08:45:43.535097000 UTC +00:00, updated_at: Wed, 14 Jul 2021 08:45:43.535097000 UTC +00:00>,
#<Industry:0x00005603efd59ff0 id: 66, name: "Ngành khác", created_at: Wed, 14 Jul 2021 08:45:43.545318000 UTC +00:00, updated_at: Wed, 14 Jul 2021 08:45:43.545318000 UTC +00:00>]
lib/tasks/web_crawler.rake
require 'open-uri' require 'open-uri'
require 'csv' require 'csv'
require 'zip' require 'zip'
require "rails/all"
namespace :crawler do namespace :crawler do
desc 'importjob' desc 'crawler from CareerBuilder'
task jobs: :environment do task jobs: :environment do
(1..3).each do |page| total_pages = 0
if ARGV.length <= 1
ARGV.each do |a|
task a.to_sym { ; }
case a
when 'TEST'
total_pages = 5
when 'ALL'
first_page = Nokogiri::HTML(HTTParty.get('https://careerbuilder.vn/viec-lam/tat-ca-viec-lam-vi.html').body)
jobs_per_page = first_page.css('div.job-item').count
total_jobs = first_page.css('.search-result-list .job-found p').text.split(' ').first.gsub(',', '').to_i
total_pages = (total_jobs.to_f / jobs_per_page).round
else
exit
end
end
elsif
# exit
exit
end
(1..total_pages).each do |page|
parsed_page = Nokogiri::HTML(HTTParty.get("https://careerbuilder.vn/viec-lam/tat-ca-viec-lam-trang-#{page}-vi.html").body) parsed_page = Nokogiri::HTML(HTTParty.get("https://careerbuilder.vn/viec-lam/tat-ca-viec-lam-trang-#{page}-vi.html").body)
jobs_item = parsed_page.css('div.job-item .job_link') jobs_item = parsed_page.css('div.job-item .job_link')
puts page
jobs_item.each do |item| jobs_item.each do |item|
job_page = Nokogiri::HTML(HTTParty.get('https://careerbuilder.vn/vi/tim-viec-lam/' + job_page = Nokogiri::HTML(HTTParty.get('https://careerbuilder.vn/vi/tim-viec-lam/' +
CGI.escape(item.attribute('href').text.remove('https://careerbuilder.vn/vi/tim-viec-lam/'))).body) CGI.escape(item.attribute('href').text.remove('https://careerbuilder.vn/vi/tim-viec-lam/'))).body)
...@@ -100,7 +122,9 @@ namespace :crawler do ...@@ -100,7 +122,9 @@ namespace :crawler do
industry = part.text.squish.strip industry = part.text.squish.strip
industry_list << industry industry_list << industry
end end
p industry_list industry_list.each do |industry|
Industry.create!(name: industry)
end
end end
desc 'crawler city form CareerBuilder' desc 'crawler city form CareerBuilder'
...@@ -122,6 +146,11 @@ namespace :crawler do ...@@ -122,6 +146,11 @@ namespace :crawler do
} }
city_list << city city_list << city
end end
puts city_list city_list.each do |city|
City.create!(
name: city[:name],
region: city[:region]
)
end
end end
end end
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment