Skip to content

V
venjob
Commit 784ae619 by Van Hau Le
Options

Merge branch 'page' into 'master' 

Page

See merge request !1
parents 363e2ef7 c4f83312
Pipeline #859 canceled with stages
in 0 seconds
Showing with 599 additions and 53 deletions
.~lock.jobs.csv# deleted 100644 → 0
,phuocht,devops-OptiPlex-3010,23.07.2020 08:12,file:///home/phuocht/.config/libreoffice/4;
\ No newline at end of file
Gemfile
......@@ -27,13 +27,13 @@ gem 'jbuilder', '~> 2.5'
# gem 'redis', '~> 4.0'
# Use ActiveModel has_secure_password
# gem 'bcrypt', '~> 3.1.7'
gem 'whenever', require: false
# Use ActiveStorage variant
# gem 'mini_magick', '~> 4.8'
gem 'rubocop', '~> 0.88.0', require: false
# Use Capistrano for deployment
# gem 'capistrano-rails', group: :development
gem 'will_paginate', '3.3.0'
# Reduces boot times through caching; required in config/boot.rb
gem 'bootsnap', '>= 1.1.0', require: false
......@@ -44,6 +44,7 @@ end
group :development do
# Access an interactive console on exception pages or by calling 'console' anywhere in the code.
gem 'pry'
gem 'web-console', '>= 3.3.0'
gem 'listen', '>= 3.0.5', '< 3.2'
# Spring speeds up development by keeping your application running in the background. Read more: https://github.com/rails/spring
......
Gemfile.lock
......@@ -47,6 +47,7 @@ GEM
archive-zip (0.12.0)
io-like (~> 0.3.0)
arel (9.0.0)
ast (2.4.1)
bindex (0.8.1)
bootsnap (1.4.6)
msgpack (~> 1.0)
......@@ -64,6 +65,8 @@ GEM
chromedriver-helper (2.1.1)
archive-zip (~> 0.10)
nokogiri (~> 1.8)
chronic (0.10.2)
coderay (1.1.3)
coffee-rails (4.2.2)
coffee-script (>= 2.2.0)
railties (>= 4.0.0)
......@@ -125,6 +128,12 @@ GEM
nokogiri (1.10.10)
mini_portile2 (~> 2.4.0)
ntlm-http (0.1.1)
parallel (1.19.2)
parser (2.7.1.4)
ast (~> 2.4.1)
pry (0.13.1)
coderay (~> 1.1)
method_source (~> 1.0)
public_suffix (4.0.5)
puma (3.12.6)
rack (2.2.3)
......@@ -154,11 +163,25 @@ GEM
method_source
rake (>= 0.8.7)
thor (>= 0.19.0, < 2.0)
rainbow (3.0.0)
rake (13.0.1)
rb-fsevent (0.10.4)
rb-inotify (0.10.1)
ffi (~> 1.0)
regexp_parser (1.7.1)
rexml (3.2.4)
rubocop (0.88.0)
parallel (~> 1.10)
parser (>= 2.7.1.1)
rainbow (>= 2.2.2, < 4.0)
regexp_parser (>= 1.7)
rexml
rubocop-ast (>= 0.1.0, < 1.0)
ruby-progressbar (~> 1.7)
unicode-display_width (>= 1.4.0, < 2.0)
rubocop-ast (0.2.0)
parser (>= 2.7.0.1)
ruby-progressbar (1.10.1)
ruby_dep (1.5.0)
rubyzip (2.3.0)
sass (3.7.4)
......@@ -199,6 +222,7 @@ GEM
unf (0.1.4)
unf_ext
unf_ext (0.0.7.7)
unicode-display_width (1.7.0)
web-console (3.7.0)
actionview (>= 5.0)
activemodel (>= 5.0)
......@@ -208,6 +232,9 @@ GEM
websocket-driver (0.7.3)
websocket-extensions (>= 0.1.0)
websocket-extensions (0.1.5)
whenever (1.0.0)
chronic (>= 0.6.3)
will_paginate (3.3.0)
xpath (3.2.0)
nokogiri (~> 1.8)
......@@ -224,8 +251,10 @@ DEPENDENCIES
listen (>= 3.0.5, < 3.2)
mechanize (~> 2.7.6)
mysql2 (~> 0.5.3)
pry
puma (~> 3.11)
rails (~> 5.2.4, >= 5.2.4.3)
rubocop (~> 0.88.0)
sass-rails (~> 5.0)
selenium-webdriver
spring
......@@ -234,6 +263,8 @@ DEPENDENCIES
tzinfo-data
uglifier (>= 1.3.0)
web-console (>= 3.3.0)
whenever
will_paginate (= 3.3.0)
RUBY VERSION
ruby 2.6.6p146
......
app/assets/javascripts/cities_area.js 0 → 100644
$(document).ready(function(){
$("div.viet-nam").click(function(){
$(window).scrollTop($('.vietnam-area').offset().top);
});
$("div.international").click(function(){
$(window).scrollTop($('.international-area').offset().top);
});
});
app/assets/javascripts/solid.js 0 → 100644
This source diff could not be displayed because it is too large. You can view the blob instead.
app/assets/javascripts/solid.min.js 0 → 100644
This source diff could not be displayed because it is too large. You can view the blob instead.
app/assets/stylesheets/custom.css.scss → app/assets/stylesheets/all_page.css.scss
html {
overflow-y: scroll;
padding: 0px;
margin: 0px;
}
body {
padding-top: 60px;
}
section {
overflow: auto;
}
......@@ -48,7 +44,7 @@ p {
font-size: 1.1em;
line-height: 1.7em;
}
#logo{
height: 200px;
width: 100px;
.footer-text{
height: 100px;
position: relative;
}
app/assets/stylesheets/cities.scss 0 → 100644
.city-banner{
background: linear-gradient(to top, #33ccff 0%, #006600 100%);
font-size: 38px;
text-align: center;
color: white;
font-style: italic;
font-family: Bookman, URW Bookman L, serif;
}
.city-name {
margin: 10px;
padding: 10px;
text-align: center;
}
.remove-decoration {
&:hover{
a {
text-decoration: none;
}
}
}
.viet-nam, .international {
cursor: pointer;
}
app/assets/stylesheets/top_pages.scss 0 → 100644
// Place all the styles related to the Top_pages controller here.
// They will automatically be included in application.css.
// You can use Sass (SCSS) here: http://sass-lang.com/
.float-right {
position: absolute;
bottom: 0;
right: 5%;
}
.top-page{
text-align: center;
width: 100%;
height: 100%;
}
.total-job{
position: absolute;
color: white;
top: 40%;
right: 30%;
font-size: 60px;
}
.banner-ground{
position: relative;
width: 100%;
height: 50vh;
}
.top-banner {
background-image: url('banner.jpg');
width: 100%;
height: 100%;
background-repeat: no-repeat;
background-size: cover;
font-style: italic;
font-family: Bookman, URW Bookman L, serif;
}
.salary{
color: green;
}
.job-details{
position: relative;
padding: 15px;
margin: 10px;
}
#button-follow{
position: absolute;
top: 20%;
right: 5%;
}
.row-table{
color: black;
padding: 15px;
margin: 15px;
text-align: center;
}
.city-banner{
background: linear-gradient(to top, #33ccff 0%, #006600 100%);
font-size: 38px;
text-align: center;
color: white;
font-style: italic;
font-family: Bookman, URW Bookman L, serif;
}
.industry-banner{
background: linear-gradient(to top, #33ccff 0%, #006666 100%);
font-size: 38px;
text-align: center;
color: white;
font-style: italic;
font-family: Bookman, URW Bookman L, serif;
}
.search-bar{
size: 50;
}
.custom_footer {
width: 100%;
height: 150px;
background-image: linear-gradient(160deg, black, #8c8686);
color: white;
}
.city-list:hover, .industry-list:hover{
background-color: black;
.city-name, .count-job, .industry-name{
color: white;
}
}
.city-list, .industry-list{
&:hover {
a {
text-decoration: none;
}
}
}
.city-name, .count-job, .industry-name{
text-decoration: none;
color: black;
}
.search-bar{
padding: 20px;
}
.all-city:hover, .all-industry:hover{
.all-cities, .all-industries{
color: blue;
}
}
.all-city, .all-industry{
&:hover{
a {
text-decoration: none;
}
}
}
app/controllers/cities_controller.rb 0 → 100644
class CitiesController < ApplicationController
def index
@cities_vietnam = City.all_city.vietnam
@cities_international = City.all_city.international
end
end
app/controllers/industries_controller.rb 0 → 100644
class IndustriesController < ApplicationController
end
app/controllers/jobs_controller.rb
class JobsController < ApplicationController
def index
end
end
app/controllers/top_pages_controller.rb 0 → 100644
class TopPagesController < ApplicationController
def index
@cities = City.all
@industries = Industry.all
@total_jobs = Job.count
@jobs = Job.limit(5).order(created_at: :desc)
@jobs_of_cities = City.top_city
@jobs_of_industries = Industry.top_industry
end
end
app/helpers/application_helper.rb
module ApplicationHelper
def full_title(page_title)
page_title.to_s
end
end
app/models/city.rb
class City < ApplicationRecord
has_many :city_jobs
has_many :jobs, through: :city_jobs
scope :all_city, -> { joins(:jobs).group(:city_id).order('count(job_id) DESC') }
scope :vietnam, -> { where('location = 1') }
scope :international, -> { where('location = 0') }
def self.top_city
joins(:jobs).group(:city_id).order('count(job_id) DESC').limit(9)
end
end
app/models/city_job.rb
class CityJob < ApplicationRecord
belongs_to :city
belongs_to :job
end
app/models/industry.rb
class Industry < ApplicationRecord
has_many :industry_jobs
has_many :jobs, through: :industry_jobs
def self.top_industry
joins(:jobs).group(:industry_id).order('count(job_id) DESC').limit(9)
end
end
app/models/industry_job.rb
class IndustryJob < ApplicationRecord
belongs_to :industry
belongs_to :job
end
app/models/job.rb
......@@ -14,4 +14,9 @@ class Job < ApplicationRecord
has_many :histories
has_many :users, through: :histories
def company_name
@company_name ||= company&.name
end
end
app/views/cities/_cities_international.html.erb 0 → 100644
<% @cities_international.each do |city| %>
<div class="col-3 remove-decoration">
<%= link_to '#' do %>
<div class="border border-dark rounded city-name">
<div>
<strong><%= city.name %></strong>
</div>
<div>
<%= city.jobs.count %>
</div>
</div>
<% end %>
</div>
<% end %>
app/views/cities/_cities_vietnam.html.erb 0 → 100644
<% @cities_vietnam.each do |city| %>
<div class="col-3 remove-decoration">
<%= link_to '#' do %>
<div class="border border-dark rounded city-name">
<div>
<strong><%= city.name %></strong>
</div>
<div>
<%= city.jobs.count %>
</div>
</div>
<% end %>
</div>
<% end %>
app/views/cities/index.html.erb 0 → 100644
<% provide(:title, 'All Cities') %>
<div class="container">
<div class="all-cities">
<div class="city-banner rounded">City list</div>
<div class="row">
<div class="col-6">
<div class="row-table border border-dark rounded viet-nam">Việt Nam</div>
</div>
<div class="col-6">
<div class="row-table border border-dark rounded international">International</div>
</div>
</div>
</div>
<div class="city-banner rounded vietnam-area">Việt Nam</div>
<div class="city-vietnam">
<div class="row">
<%= render 'cities/cities_vietnam' %>
</div>
</div>
<div class="city-banner rounded international-area">International</div>
<div class="city-international">
<div class="row">
<%= render 'cities/cities_international' %>
</div>
</div>
</div>
app/views/layouts/_footer.html.erb 0 → 100644
<div class="custom_footer">
<div class="footer-text">
<div class="float-right">
Copyright © ZIGExN VeNtura 2020
</div>
</div>
</div>
app/views/layouts/_header.html.erb 0 → 100644
<header class="navbar navbar-expand-lg navbar-dark bg-dark">
<div class="collapse navbar-collapse">
<div class="container-fluid">
<%= link_to image_tag("logo_venjob.png", alt: "Logo", id: "logo_venjob", width: 120), root_path %>
<nav>
<ul class="navbar-nav mr-auto">
<li><%= link_to "Login", '#' , class: "nav-item nav-link" %></li>
<li><%= link_to "Register", '#', class: "nav-item nav-link" %></li>
<li><%= link_to "Favorite", '#', class: "nav-item nav-link" %></li>
<li><%= link_to "History", '#', class: "nav-item nav-link" %></li>
</ul>
</nav>
</div>
</div>
</header>
app/views/layouts/_search_bar.html.erb 0 → 100644
<div class="search-bar">
<div class="row">
<div class="col-sm">
<input class="form-control mr-sm-2" type="search" placeholder="Search" aria-label="Search">
</div>
<div class="col-sm">
<select class="form-control" id="exampleFormControlSelect1">
<option>--None</option>
<%= @cities.each do |city| %>
<option><%= city.name %></option>
<% end %>
</select>
</div>
<div class="col-sm">
<select class="form-control" id="exampleFormControlSelect1">
<option>--None</option>
<%= @industries.each do |industry| %>
<option><%= industry.name %></option>
<% end %>
</select>
</div>
<div class="col-2-sm">
<button class="btn btn-primary" type="submit">Search</button>
</div>
</div>
</div>
app/views/layouts/_show_cities.html.erb 0 → 100644
<% @jobs_of_cities.each do |city| %>
<div class="col-4">
<div class="row-table border border-dark rounded city-list">
<%= link_to '#' do %>
<div class="city-name"><strong><%= city.name %></strong></div>
<div class="count-job"><%= city.jobs.count %></div>
<% end %>
</div>
</div>
<% end %>
app/views/layouts/_show_industries.html.erb 0 → 100644
<% @jobs_of_industries.each do |industry| %>
<div class="col-4">
<div class="row-table border border-dark rounded industry-list">
<%= link_to '#' do %>
<div class="industry-name"><strong><%= industry.name %></strong></div>
<div class="count-job"><%= industry.jobs.count %></div>
<% end %>
</div>
</div>
<% end %>
app/views/layouts/_show_jobs.html.erb 0 → 100644
<% @jobs.each do |job| %>
<% if job.cities.present? %>
<div class="border border-dark rounded">
<div class="job-details">
<div class="title"><strong><%= job.title %></strong></div>
<div><%= job.company_name %></div>
<div class="salary">Salary: <%= job.salary %></div>
<div>▼
<% job.cities.each do |city| %>
<%= city.name %>
<% end %>
</div>
<button type="button" class="btn btn-primary" id="button-follow">♥ Follow</button>
</div>
</div>
<br>
<% end %>
<% end %>
app/views/layouts/application.html.erb
<!DOCTYPE html>
<html>
<head>
<title>Venjob</title>
<title><%= full_title(yield(:title)) %></title>
<%= csrf_meta_tags %>
<%= csp_meta_tag %>
......@@ -12,23 +12,8 @@
<%= javascript_include_tag 'bootstrap', media: 'all', 'data-turbolinks-track': 'reload' %>
</head>
<body>
<header class="navbar navbar-expand-lg navbar-dark bg-dark">
<div class="collapse navbar-collapse">
<div class="container">
<%= link_to image_tag("logo_venjob.png", alt: "Logo", id: "logo_venjob", width: 120, left: 0), '#'%>
<nav>
<ul class="navbar-nav mr-auto">
<li><%= link_to "Login", '#' %></li>
<li><%= link_to "Register", '#' %></li>
<li><%= link_to "Favorite", '#' %></li>
<li><%= link_to "History", '#' %></li>
</ul>
</nav>
</div>
</div>
</header>
<div class="container">
<%= render "layouts/header" %>
<%= yield %>
</div>
<%= render "layouts/footer" %>
</body>
</html>
app/views/top_pages/index.html.erb 0 → 100644
<% provide(:title, 'Venjob') %>
<div class="banner-ground">
<div class="top-banner">
<div class="total-job">Having <%= @total_jobs %> jobs for you!</div>
</div>
</div>
<div class="container">
<div class="search-bar"><%= render 'layouts/search_bar' %></div>
<div class="job-list"><%= render 'layouts/show_jobs' %></div>
<div class="city-banner rounded">City</div>
<div class="row"><%= render 'layouts/show_cities' %></div>
<div class="all-city">
<%= link_to '#' do %>
<div class="row-table border border-dark rounded all-cities">
<strong>All Cities</strong>
</div>
<% end %>
</div>
<div class="industry-banner rounded">Industry</div>
<div class="row"><%= render 'layouts/show_industries' %></div>
<div class="all-industry">
<%= link_to '#' do %>
<div class="row-table border border-dark rounded all-industries">
<strong>All Industries</strong>
</div>
<% end %>
</div>
</div>
config/routes.rb
Rails.application.routes.draw do
# For details on the DSL available within this file, see http://guides.rubyonrails.org/routing.html
resources :jobs
resources :top_pages
resources :industries
resources :cities
root to: "top_pages#index"
end
config/schedule.rb 0 → 100644
env :PATH, ENV['PATH']
every 20.minutes do
rake 'import:auto'
end
db/schema.rb
......@@ -10,7 +10,7 @@
#
# It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema.define(version: 2020_07_15_035356) do
ActiveRecord::Schema.define(version: 2020_07_23_071239) do
create_table "cities", options: "ENGINE=InnoDB DEFAULT CHARSET=utf8", force: :cascade do |t|
t.string "name"
......@@ -83,6 +83,11 @@ ActiveRecord::Schema.define(version: 2020_07_15_035356) do
t.datetime "updated_at", null: false
end
create_table "top_pages", options: "ENGINE=InnoDB DEFAULT CHARSET=utf8", force: :cascade do |t|
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
end
create_table "users", options: "ENGINE=InnoDB DEFAULT CHARSET=utf8", force: :cascade do |t|
t.string "name"
t.string "email"
......
jobs.csv
This source diff could not be displayed because it is too large. You can view the blob instead.
lib/csv/jobs.csv 0 → 100644
This source diff could not be displayed because it is too large. You can view the blob instead.
lib/src/csv_importer.rb 0 → 100644
require 'net/ftp'
require 'csv'
require 'zip'
class CSVImporter
NAME_DOMAIN = '192.168.1.156'.freeze
FTP_USERNAME = 'training'.freeze
FTP_PASSWORD = 'training'.freeze
def initialize(logger)
@logger = logger
@extracting_directory = Rails.root.join('lib', 'csv')
@zip_directory = Rails.root.join('jobs.zip')
@importer = Rails.root.join('lib', 'csv', 'jobs.csv')
end
def import
get_file_csv
extract_zip
import_file_csv
end
def get_file_csv
Net::FTP.open(NAME_DOMAIN, USERNAME_FTP, PASSWORD_FTP) do |ftp|
ftp.getbinaryfile('jobs.zip')
end
end
def extract_zip
FileUtils.mkdir_p(@extracting_directory)
Zip::File.open(@zip_directory) do |zip_file|
zip_file.each do |f|
fpath = File.join(@extracting_directory, f.name)
zip_file.extract(f, fpath) unless File.exist?(fpath)
end
end
end
def import_file_csv
CSV.foreach(@importer, headers: true) do |row|
begin
company_name = row["company name"]
next if company_name.blank?
company_address = row["company address"]
company_introduction = row["benefit"]
company = Company.find_or_create_by!(name: company_name,
address: company_address,
introduction: company_introduction)
title_job = row["name"]
next if title_job.blank?
description_job = "#{row["description"]} #{row["requirement"]}"
level = row["level"]
salary = row["salary"]
job = Job.find_or_create_by!(title: title_job,
description: description_job,
level: level,
salary: salary,
company_id: company.id)
industry_name = row["category"]
industries_relationship = Industry.where(name: industry_name)
next if industries_relationship.blank?
job.industries << industries_relationship
location_data = row["work place"]
location = location_data.gsub('["', '').gsub('"]', '')
location_relationship = City.where(name: location)
next if location_relationship.blank?
job.cities << location_relationship
rescue StandardError => e
@logger.error e.message
end
end
end
end
lib/src/job_parser.rb 0 → 100644
class JobParser
def initialize(logger, url)
@logger = logger
@url = url
end
def crawl_all
find_company
find_job
end
def find_company
info = Nokogiri::HTML(URI.open(@url))
links = info.css('div.caption a.company-name').map { |link| link['href'] }
links.each do |link|
next if link == 'javascript:void(0);'
page = Nokogiri::HTML(URI.open(URI.escape(link)))
name = page.search('p.name')&.text
next if name.blank?
address = page.css('div.content p').children[1]&.text
introduction = page.css('div.main-about-us').text
begin
Company.find_or_create_by!(name: name,
address: address,
introduction: introduction)
rescue StandardError => e
@logger.error e.message
end
end
end
def city_relationship(row, job)
location_relationship = row.css('div.map p a').children.map { |name_city| name_city.text.strip }
cities_relationship = City.where(name: location_relationship)
job.cities << cities_relationship
end
def industry_relationship(row, job)
industry_relationship = row.css('li a').children.map { |name_industry| name_industry.text.strip }
industries_relationship = Industry.where(name: industry_relationship)
job.industries << industries_relationship
end
def create_job(title, link_page, row, company)
description = link_page.search('div.detail-row').to_s
salary = row.at_xpath('//li[./strong/i[contains(@class, "fa fa-usd")]]/p').text.strip
experience = row.at_xpath('//li[./strong/i[contains(@class, "fa fa-briefcase")]]/p')&.text&.strip
level = row.at_xpath('//li[./strong/i[contains(@class, "mdi mdi-account")]]/p').text.strip
expiration_date = row.at_xpath('//li[./strong/i[contains(@class, "mdi mdi-calendar-check")]]/p').text.strip
job = Job.find_or_create_by!(title: title,
level: level,
salary: salary,
experience: experience,
expiration_date: expiration_date,
description: description,
company_id: company.id)
city_relationship(row, job)
industry_relationship(row, job)
end
def find_job
info = Nokogiri::HTML(URI.open(@url))
link = info.css('a.job_link').map { |link| link['href'] }
link.each do |link|
link_page = Nokogiri::HTML(URI.open(URI.escape(link)))
row = link_page.search('div.bg-blue div.row')
next if row.blank?
begin
company_name = link_page.search('div.job-desc a.job-company-name').text.strip
company = Company.find_by(name: company_name)
next if company.blank?
title = link_page.search('div.job-desc p').text.strip
next if title.blank?
create_job(title, link_page, row, company)
rescue StandardError => e
@logger.error e.message
end
end
end
end
lib/tasks/crawler_import.rake
require 'src/crawler.rb'
require 'net/ftp'
require 'csv'
require 'zip'
action = Crawler.new
require 'src/job_parser.rb'
require 'src/csv_importer.rb'
namespace :import do
desc "crawler data"
desc 'crawler data'
task crawler: :environment do
action.crawl_city
action.crawl_industry
action.crawl_company
action.crawl_job_relationships
action = Crawler.new(logger, url).crawl_city_industry
end
desc "get file CSV from server"
task csv_get: :environment do
action.get_file_csv
action.extract_zip('./jobs.zip','.')
desc 'Crontab'
task auto: :environment do
crontab = JobParser.new(logger, url)
csv_importer = CSVImporter.new(logger)
crontab.crawl_all
csv_importer.import
end
desc "Import data from CSV"
task data_csv: :environment do
action.import_file_csv
def logger
Logger.new(Rails.root.join('log','crawling.log'))
end
end
\ No newline at end of file
def url
'https://careerbuilder.vn/viec-lam/tat-ca-viec-lam-trang-1-vi.html'.freeze
end
end
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment