Skip to content

V
venjob
Commit 0bdc8e73 by Huỳnh Thiên Phước
Options

rename variable, create import csv file

parent d58cf3ff
Pipeline #766 canceled with stages
in 0 seconds
Showing with 147 additions and 56 deletions
app/assets/stylesheets/top_pages.scss
...@@ -78,6 +78,14 @@ ...@@ -78,6 +78,14 @@
background-image: linear-gradient(160deg, black, #8c8686); background-image: linear-gradient(160deg, black, #8c8686);
color: white; color: white;
} }
.city-name{ .city-list:hover{
background-color: black;
.city-name, .count-job{
text-decoration: none;
color: white;
}
}
.city-name, .count-job{
text-decoration: none;
color: black;
} }
app/controllers/top_pages_controller.rb
class TopPagesController < ApplicationController class TopPagesController < ApplicationController
def index def index
@total_jobs = Job.ids @total_jobs = Job.count
@jobs = Job.limit(5).order(created_at: :desc) @jobs = Job.limit(5).order(created_at: :desc)
@jobs_of_cities = CityJob.limit(9).group('city_id').order('Count(*) DESC').count @jobs_of_cities = CityJob.top_city
@jobs_of_industries = IndustryJob.limit(9).group('industry_id').order('Count(*) DESC').count @jobs_of_industries = IndustryJob.limit(9).group('industry_id').order('Count(*) DESC').count
end end
end end
app/helpers/application_helper.rb
module ApplicationHelper module ApplicationHelper
def full_title(page_title) def full_title(page_title)
"#{page_title}" page_title.to_s
end end
end end
app/models/city_job.rb
...@@ -2,4 +2,7 @@ class CityJob < ApplicationRecord ...@@ -2,4 +2,7 @@ class CityJob < ApplicationRecord
belongs_to :city belongs_to :city
belongs_to :job belongs_to :job
def self.top_city
limit(9).group('city_id').order('Count(*) DESC').count
end
end end
app/models/industry_job.rb
class IndustryJob < ApplicationRecord class IndustryJob < ApplicationRecord
belongs_to :industry belongs_to :industry
belongs_to :job belongs_to :job
end end
app/views/layouts/_show_cities.html.erb
<% @jobs_of_cities.each do |city, count_job| %> <% @jobs_of_cities.each do |city, count_job| %>
<div class="col-4"> <div class="col-4">
<div class="row-table border border-dark rounded"> <div class="row-table border border-dark rounded city-list">
<div><strong><%= link_to "#{City.find(city).name}", '#', class: "city-name" %></strong></div> <%= link_to '#' do%>
<div class="city-name"><strong><%= City.find(city).name %></strong></div>
<div class="count-job"><%= count_job %></div> <div class="count-job"><%= count_job %></div>
<% end %>
</div> </div>
</div> </div>
<% end %> <% end %>
\ No newline at end of file
app/views/layouts/_show_jobs.html.erb
...@@ -10,7 +10,6 @@ ...@@ -10,7 +10,6 @@
<%= city.name %> <%= city.name %>
<% end %> <% end %>
</div> </div>
<% job.description.html_safe %>
<button type="button" class="btn btn-primary" id="button-follow">♥ Follow</button> <button type="button" class="btn btn-primary" id="button-follow">♥ Follow</button>
</div> </div>
</div> </div>
......
app/views/top_pages/index.html.erb
<% provide(:title, 'Venjob') %> <% provide(:title, 'Venjob') %>
<div class="banner-ground"> <div class="banner-ground">
<div class="top-banner"> <div class="top-banner">
<div class="total-job">Having <%= @total_jobs.count %> jobs for you!</div> <div class="total-job">Having <%= @total_jobs %> jobs for you!</div>
</div> </div>
</div> </div>
<div class="search-bar"><%= render 'layouts/search_bar' %></div>
<br>
<div class="container"> <div class="container">
<div class="search-bar"><%= render 'layouts/search_bar' %></div>
<br>
<div class="job-list"><%= render 'layouts/show_jobs' %></div> <div class="job-list"><%= render 'layouts/show_jobs' %></div>
</div>
<div class="city-banner">City</div> <div class="city-banner rounded">City</div>
<div class="container">
<div class="row"><%= render 'layouts/show_cities' %></div> <div class="row"><%= render 'layouts/show_cities' %></div>
<div class="all-industry"> <div class="all-city">
<div class="row-table border border-dark rounded">All Cities</div> <div class="row-table border border-dark rounded"><strong>All Cities</strong></div>
</div> </div>
</div> <div class="industry-banner rounded">Industry</div>
<div class="industry-banner">Industry</div>
<div class="container">
<div class="row"><%= render 'layouts/show_industries' %></div> <div class="row"><%= render 'layouts/show_industries' %></div>
<div class="all-industry"> <div class="all-industry">
<div class="row-table border border-dark rounded">All Industries</div> <div class="row-table border border-dark rounded"><strong>All Industries</strong></div>
</div> </div>
</div> </div>
lib/src/csvimporter.rb 0 → 100644
require 'net/ftp'
require 'csv'
require 'zip'
class CSVimporter
def initialize(logger)
@logger = logger
@NAME_DOMAIN = '192.168.1.156'
@USERNAME_FTP = 'training'
@PASSWORD_FTP = 'training'
@extracting_directory = Rails.root.join('lib', 'csv')
@zip_directory = Rails.root.join('jobs.zip')
@importer = Rails.root.join('lib', 'csv', 'jobs.csv')
end
def import
get_file_csv
extract_zip
import_file_csv
end
def get_file_csv
Net::FTP.open(@NAME_DOMAIN, @USERNAME_FTP, @PASSWORD_FTP) do |ftp|
ftp.getbinaryfile('jobs.zip')
end
end
def extract_zip
FileUtils.mkdir_p(@extracting_directory)
Zip::File.open(@zip_directory) do |zip_file|
zip_file.each do |f|
fpath = File.join(@extracting_directory, f.name)
zip_file.extract(f, fpath) unless File.exist?(fpath)
end
end
end
def import_file_csv
CSV.foreach(@importer, headers: true) do |row|
begin
company_name = row["company name"]
next if company_name.blank?
company_address = row["company address"]
company_introduction = row["benefit"]
company = Company.find_or_create_by!(name: company_name,
address: company_address,
introduction: company_introduction)
title_job = row["name"]
next if title_job.blank?
description_job = "#{row["description"]} #{row["requirement"]}"
level = row["level"]
salary = row["salary"]
job = Job.find_or_create_by!(title: title_job,
description: description_job,
level: level,
salary: salary,
company_id: company.id)
industry_name = row["category"]
industries_relationship = Industry.where(name: industry_name)
if industries_relationship.blank?
industry = Industry.create!(name: industry_name)
industries_relationship = Industry.where(name: industry)
job.industries << industries_relationship
else
job.industries << industries_relationship
end
location_data = row["work place"]
location = location_data.gsub('["', '').gsub('"]', '')
location_relationship = City.where(name: location)
if location_relationship.blank?
city = City.create!(name: location)
location_relationship = City.where(name: city)
job.cities << location_relationship
else
job.cities << location_relationship
end
rescue StandardError => e
@logger.error e.message
end
end
end
end
\ No newline at end of file
lib/src/crontab.rb → lib/src/jobparser.rb
require 'net/ftp' class JobParser
require 'csv'
require 'zip'
class InforJob
def initialize(logger, url) def initialize(logger, url)
@mylogger = logger @logger = logger
@url = url @url = url
end end
...@@ -25,49 +22,46 @@ class InforJob ...@@ -25,49 +22,46 @@ class InforJob
address = page.css('div.content p').children[1]&.text address = page.css('div.content p').children[1]&.text
introduction = page.css('div.main-about-us').text introduction = page.css('div.main-about-us').text
begin begin
puts name
Company.find_or_create_by!(name: name, Company.find_or_create_by!(name: name,
address: address, address: address,
introduction: introduction) introduction: introduction)
rescue StandardError => e rescue StandardError => e
@mylogger.error e.message @logger.error e.message
end end
end end
end end
def create_city_rel(row, info_job) def city_relationship(row, job)
location_rel = row.css('div.map p a').children.map { |name_city| name_city.text.strip } location_relationship = row.css('div.map p a').children.map { |name_city| name_city.text.strip }
city_table = City.where(name: location_rel) cities_relationship = City.where(name: location_relationship)
puts "#{info_job.cities << city_table}" job.cities << cities_relationship
info_job.cities << city_table
end end
def create_industry_rel(row, info_job) def industry_relationship(row, job)
industry_rel = row.css('li a').children.map { |name_industry| name_industry.text.strip } industry_relationship = row.css('li a').children.map { |name_industry| name_industry.text.strip }
industry_table = Industry.where(name: industry_rel) industries_relationship = Industry.where(name: industry_relationship)
puts "#{info_job.industries << industry_table}" job.industries << industries_relationship
info_job.industries << industry_table
end end
def create_job(title, link_page, row, company_table) def create_job(title, link_page, row, company)
description = link_page.search('div.detail-row').to_s description = link_page.search('div.detail-row').to_s
salary = row.at_xpath('//li[./strong/i[contains(@class, "fa fa-usd")]]/p').text.strip salary = row.at_xpath('//li[./strong/i[contains(@class, "fa fa-usd")]]/p').text.strip
experience = row.at_xpath('//li[./strong/i[contains(@class, "fa fa-briefcase")]]/p')&.text&.strip experience = row.at_xpath('//li[./strong/i[contains(@class, "fa fa-briefcase")]]/p')&.text&.strip
level = row.at_xpath('//li[./strong/i[contains(@class, "mdi mdi-account")]]/p').text.strip level = row.at_xpath('//li[./strong/i[contains(@class, "mdi mdi-account")]]/p').text.strip
expiration_date = row.at_xpath('//li[./strong/i[contains(@class, "mdi mdi-calendar-check")]]/p').text.strip expiration_date = row.at_xpath('//li[./strong/i[contains(@class, "mdi mdi-calendar-check")]]/p').text.strip
info_job = Job.find_or_create_by!(title: title, job = Job.find_or_create_by!(title: title,
level: level, level: level,
salary: salary, salary: salary,
experience: experience, experience: experience,
expiration_date: expiration_date, expiration_date: expiration_date,
description: description, description: description,
company_id: company_table.id) company_id: company.id)
create_city_rel(row, info_job) city_relationship(row, job)
create_industry_rel(row, info_job) industry_relationship(row, job)
end end
def find_job def find_job
...@@ -79,19 +73,18 @@ class InforJob ...@@ -79,19 +73,18 @@ class InforJob
next if row.blank? next if row.blank?
begin begin
name_company = link_page.search('div.job-desc a.job-company-name').text.strip company_name = link_page.search('div.job-desc a.job-company-name').text.strip
company_table = Company.find_by(name: name_company) company = Company.find_by(name: company_name)
next if company_table.blank? next if company.blank?
title = link_page.search('div.job-desc p').text.strip title = link_page.search('div.job-desc p').text.strip
next if title.blank? next if title.blank?
create_job(title, link_page, row, company_table) create_job(title, link_page, row, company)
rescue StandardError => e rescue StandardError => e
puts e @logger.error e.message
# @mylogger.error e.message
end end
end end
end end
end end
lib/tasks/crawler_import.rake
require 'src/crawler.rb' require 'src/crawler.rb'
require 'src/crontab.rb' require 'src/jobparser.rb'
require 'src/csvimporter.rb'
namespace :import do namespace :import do
desc 'crawler data' desc 'crawler data'
...@@ -9,12 +10,10 @@ namespace :import do ...@@ -9,12 +10,10 @@ namespace :import do
end end
desc 'Crontab' desc 'Crontab'
task auto: :environment do task auto: :environment do
action = Crawler.new(logger) crontab = JobParser.new(logger, url)
crontab = InforJob.new(logger, url) csvimporter = CSVimporter.new(logger)
crontab.crawl_all crontab.crawl_all
action.get_file_csv csvimporter.import
action.extract_zip('./jobs.zip', 'lib/csv')
action.import_file_csv(Rails.root.join('lib', 'csv', 'jobs.csv'))
end end
def logger def logger
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment