Commit 043ca43e by Ngô Trung Hưng

fix crawler

parent a0abd223
Pipeline #751 failed with stages
in 0 seconds
$(document).on("turbolinks:load", function(){
// => scroll to top
$(window).scroll(function(){
if ($(this).scrollTop() > 550) {
$(".btn-scroll-top").css('opacity',1);
$(".btn-scroll-top").fadeIn(400);
} else {
$(".btn-scroll-top").fadeOut(400);
}
});
$(".btn-scroll-top").click(function (e){
var body = $("html, body");
console.log('wed')
body.stop().animate({ scrollTop: 0}, 700, 'swing')
});
// => end
$('.block_click_favorite').click(function (e) {
$('.overlay_login').addClass('show');
});
$('.icon_close_modal_login').click(function (e) {
$('.overlay_login').removeClass('show');
});
});
# Place all the behaviors and hooks related to the matching controller here.
# All this logic will automatically be available in application.js.
# You can use CoffeeScript in this file: http://coffeescript.org/
# Place all the behaviors and hooks related to the matching controller here.
# All this logic will automatically be available in application.js.
# You can use CoffeeScript in this file: http://coffeescript.org/
.box_link_favotite {
position: relative;
width: 100%;
height: 100%;
padding: 10px 10px;
span {
color: #999;
}
}
.block_link_favorite :hover {
color: #e62958;
}
.block_link_favorite.md {
display: none;
}
.link_favorite {
color: #999;
z-index: 100000;
&:hover {
text-decoration: none;
}
}
.block_click_favorite {
color: #999;
cursor: pointer;
}
.block_click_favorite.md {
display: none;
}
.link_favorite_top {
position: absolute;
z-index: 1;
// position: relative;
padding: 15px 20px;
top: 0;
right: 0;
}
.overlay_login {
visibility: hidden;
opacity: 0;
transition: 0.5;
position: fixed;
z-index: 99999;
top: 0;
bottom: 0;
left: 0;
right: 0;
transition: linear 0.2s;
background-color: rgba($color: #000000, $alpha: 0.5);
}
.overlay_login.show {
opacity: 1;
visibility: visible;
transition: linear 0.2s;
}
.modal_login {
position: absolute;
overflow: hidden;
top: 50%;
left: 50%;
height: 300px;
width: 600px;
transform: translateY(-50%) translateX(-50%);
background-color: #fff;
border-radius: 5px;
transition: cubic-bezier(0.165, 0.84, 0.44, 1);
}
// modal
.icon_close_modal_login {
cursor: pointer;
float: right;
transform: translateX(-10px);
font-size: 22px;
font-weight: 100;
color: rgba($color: #999, $alpha: 0.9);
span {
}
}
.modal_login_notice_text {
text-align: center;
color: #666;
font-size: 19px;
color: rgb(60, 71, 121);
span {
}
}
.modal_login_header {
background-color: #eee;
height: 50px;
line-height: 50px;
text-align: center;
font-size: 18px;
color: #333;
font-weight: 600;
font-family: 'Raleway', sans-serif;
}
@media only screen and (max-width: 768px) {
.block_link_favorite.lg {
display: none;
}
.block_click_favorite.lg {
display: none;
}
.block_link_favorite.md {
display: block;
}
.block_click_favorite.md {
display: block;
}
.pagination li a {
font-size: 14px !important;
}
.modal_login {
width: 95vw;
font-size: 14px;
}
.modal_login_header {
font-size: 15px;
}
.icon_close_modal_login {
font-size: 15px;
}
}
@media only screen and (max-width: 576px) {
.pagination li a {
font-size: 13px !important;
}
}
// Place all the styles related to the home controller here.
// They will automatically be included in application.css.
// You can use Sass (SCSS) here: http://sass-lang.com/
// Place all the styles related to the job controller here.
// They will automatically be included in application.css.
// You can use Sass (SCSS) here: http://sass-lang.com/
...@@ -5,7 +5,7 @@ require 'open-uri' ...@@ -5,7 +5,7 @@ require 'open-uri'
# Crawler data # Crawler data
class Crawler class Crawler
COMPANY_SECURITY = 1 COMPANY_SECURITY = 1
NUMBER_LINK = 100 NUMBER_LINK = 5
SIZE_LI = 8 SIZE_LI = 8
RANGE = 69 RANGE = 69
...@@ -51,8 +51,6 @@ class Crawler ...@@ -51,8 +51,6 @@ class Crawler
def craw_data_companies def craw_data_companies
crawl_link(NUMBER_LINK).each do |url| crawl_link(NUMBER_LINK).each do |url|
next if url.include?('javascript:void(0);')
page = safe_link(url) page = safe_link(url)
company_name = page.search('.company-info .content .name').text company_name = page.search('.company-info .content .name').text
Company.find_or_create_by(name: company_name) do |company| Company.find_or_create_by(name: company_name) do |company|
......
...@@ -44,7 +44,7 @@ class CrawlerJob < Crawler ...@@ -44,7 +44,7 @@ class CrawlerJob < Crawler
def add_data(data) def add_data(data)
id_company = Company.find_by name: data[:company_name] id_company = Company.find_by name: data[:company_name]
id_company = id_company.present? ? id_company.id : COMPANY_SECURITY id_company = id_company.present? ? id_company.id : COMPANY_SECURITY
id_job = Job.create!(name: data[:name], job = Job.create(name: data[:name],
company_id: id_company, company_id: id_company,
level: data[:level], level: data[:level],
experience: data[:exprience], experience: data[:exprience],
...@@ -52,28 +52,28 @@ class CrawlerJob < Crawler ...@@ -52,28 +52,28 @@ class CrawlerJob < Crawler
create_date: data[:created_date], create_date: data[:created_date],
expiration_date: data[:expiration_date], expiration_date: data[:expiration_date],
description: data[:description]) description: data[:description])
create_industry_relation(data[:industry_name], id_job.id) create_industry_relation(data[:industry_name], job)
create_city_relation(data[:city_name], id_job.id) create_city_relation(data[:city_name], job)
rescue StandardError => e rescue StandardError => e
logger.error "Crawler data jobs has error: #{e}" logger.error "Crawler data jobs has error: #{e}"
end end
def create_industry_relation(data, id_job) def create_industry_relation(data, job)
return if data.blank? && id_job.blank? return if data.blank? && id_job.blank?
industries = data.split(',') industries = data.split(',')
industries.each do |val| industries.each do |val|
val.gsub!('&amp;', '&') if val.include?('&amp;') val.gsub!('&amp;', '&') if val.include?('&amp;')
industry = Industry.find_or_create_by name: val.strip industry = Industry.find_or_create_by name: val.strip
IndustryJob.create(industry_id: industry.id, job_id: id_job) job.industries << industry
end end
end end
def create_city_relation(data, id_job) def create_city_relation(data, job)
cities = data.split(',') cities = data.split(',')
cities.each do |city| cities.each do |city|
city = City.find_or_create_by(name: city.strip, area: City.areas['domestic']) city = City.find_or_create_by(name: city.strip, area: City.areas['domestic'])
CityJob.create(job_id: id_job, city_id: city.id) job.cities << city
end end
end end
end end
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment