Skip to content
Projects
Groups
Snippets
Help
This project
Loading...
Sign in / Register
Toggle navigation
V
ven-job
Overview
Overview
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Trịnh Hoàng Phúc
ven-job
Commits
64410b1c
Commit
64410b1c
authored
Apr 03, 2020
by
Hoang Phuc
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Fix association, model
parent
36ba17f0
Pipeline
#544
failed with stages
in 0 seconds
Changes
8
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
Showing
8 changed files
with
44 additions
and
69 deletions
+44
-69
app/models/city_job.rb
+0
-4
app/models/industry_job.rb
+0
-4
db/migrate/20200317071357_create_jobs.rb
+3
-0
db/migrate/20200318055923_add_references_company_to_jobs.rb
+0
-5
db/migrate/20200403070221_create_join_table_city_job.rb
+4
-4
db/migrate/20200403070245_create_join_table_industry_job.rb
+4
-4
db/schema.rb
+8
-13
lib/tasks/crawler.rake
+25
-35
No files found.
app/models/city_job.rb
deleted
100644 → 0
View file @
36ba17f0
class
CityJob
<
ApplicationRecord
belongs_to
:city
belongs_to
:job
end
app/models/industry_job.rb
deleted
100644 → 0
View file @
36ba17f0
class
IndustryJob
<
ApplicationRecord
belongs_to
:industry
belongs_to
:job
end
db/migrate/20200317071357_create_jobs.rb
View file @
64410b1c
...
...
@@ -10,7 +10,10 @@ class CreateJobs < ActiveRecord::Migration[6.0]
t
.
text
:job_description
t
.
belongs_to
:company
t
.
timestamps
end
end
end
db/migrate/20200318055923_add_references_company_to_jobs.rb
deleted
100644 → 0
View file @
36ba17f0
class
AddReferencesCompanyToJobs
<
ActiveRecord
::
Migration
[
6.0
]
def
change
add_reference
:jobs
,
:company
,
null:
false
,
foreign_key:
true
end
end
db/migrate/20200
320100139_creat
e_city_job.rb
→
db/migrate/20200
403070221_create_join_tabl
e_city_job.rb
View file @
64410b1c
class
CreateCityJob
<
ActiveRecord
::
Migration
[
6.0
]
class
Create
JoinTable
CityJob
<
ActiveRecord
::
Migration
[
6.0
]
def
change
create_
table
:city_
jobs
do
|
t
|
t
.
references
:city
,
null:
false
,
foreign_key:
true
t
.
references
:job
,
null:
false
,
foreign_key:
true
create_
join_table
:cities
,
:
jobs
do
|
t
|
t
.
index
[
:city_id
,
:job_id
]
t
.
index
[
:job_id
,
:city_id
]
end
end
end
db/migrate/20200
320101302_creat
e_industry_job.rb
→
db/migrate/20200
403070245_create_join_tabl
e_industry_job.rb
View file @
64410b1c
class
CreateIndustryJob
<
ActiveRecord
::
Migration
[
6.0
]
class
Create
JoinTable
IndustryJob
<
ActiveRecord
::
Migration
[
6.0
]
def
change
create_
table
:industry_
jobs
do
|
t
|
t
.
references
:industry
,
null:
false
,
foreign_key:
true
t
.
references
:job
,
null:
false
,
foreign_key:
true
create_
join_table
:industries
,
:
jobs
do
|
t
|
t
.
index
[
:industry_id
,
:job_id
]
t
.
index
[
:job_id
,
:industry_id
]
end
end
end
db/schema.rb
View file @
64410b1c
...
...
@@ -10,7 +10,7 @@
#
# It's strongly recommended that you check this file into your version control system.
ActiveRecord
::
Schema
.
define
(
version:
2020_0
3_20_101302
)
do
ActiveRecord
::
Schema
.
define
(
version:
2020_0
4_03_070245
)
do
create_table
"applies"
,
options:
"ENGINE=InnoDB DEFAULT CHARSET=utf8"
,
force: :cascade
do
|
t
|
t
.
bigint
"user_id"
,
null:
false
...
...
@@ -27,11 +27,11 @@ ActiveRecord::Schema.define(version: 2020_03_20_101302) do
t
.
datetime
"updated_at"
,
precision:
6
,
null:
false
end
create_table
"cit
y_jobs"
,
options:
"ENGINE=InnoDB DEFAULT CHARSET=utf8"
,
force: :cascade
do
|
t
|
create_table
"cit
ies_jobs"
,
id:
false
,
options:
"ENGINE=InnoDB DEFAULT CHARSET=utf8"
,
force: :cascade
do
|
t
|
t
.
bigint
"city_id"
,
null:
false
t
.
bigint
"job_id"
,
null:
false
t
.
index
[
"city_id"
],
name:
"index_city_jobs_on_city
_id"
t
.
index
[
"job_id"
],
name:
"index_city_jobs_on_job
_id"
t
.
index
[
"city_id"
,
"job_id"
],
name:
"index_cities_jobs_on_city_id_and_job
_id"
t
.
index
[
"job_id"
,
"city_id"
],
name:
"index_cities_jobs_on_job_id_and_city
_id"
end
create_table
"companies"
,
options:
"ENGINE=InnoDB DEFAULT CHARSET=utf8"
,
force: :cascade
do
|
t
|
...
...
@@ -58,11 +58,11 @@ ActiveRecord::Schema.define(version: 2020_03_20_101302) do
t
.
datetime
"updated_at"
,
precision:
6
,
null:
false
end
create_table
"industr
y_jobs"
,
options:
"ENGINE=InnoDB DEFAULT CHARSET=utf8"
,
force: :cascade
do
|
t
|
create_table
"industr
ies_jobs"
,
id:
false
,
options:
"ENGINE=InnoDB DEFAULT CHARSET=utf8"
,
force: :cascade
do
|
t
|
t
.
bigint
"industry_id"
,
null:
false
t
.
bigint
"job_id"
,
null:
false
t
.
index
[
"industry_id"
],
name:
"index_industry_jobs_on_industry
_id"
t
.
index
[
"job_id"
],
name:
"index_industry_jobs_on_job
_id"
t
.
index
[
"industry_id"
,
"job_id"
],
name:
"index_industries_jobs_on_industry_id_and_job
_id"
t
.
index
[
"job_id"
,
"industry_id"
],
name:
"index_industries_jobs_on_job_id_and_industry
_id"
end
create_table
"jobs"
,
options:
"ENGINE=InnoDB DEFAULT CHARSET=utf8"
,
force: :cascade
do
|
t
|
...
...
@@ -73,9 +73,9 @@ ActiveRecord::Schema.define(version: 2020_03_20_101302) do
t
.
string
"salary"
t
.
string
"expiration_date"
t
.
text
"job_description"
t
.
bigint
"company_id"
t
.
datetime
"created_at"
,
precision:
6
,
null:
false
t
.
datetime
"updated_at"
,
precision:
6
,
null:
false
t
.
bigint
"company_id"
,
null:
false
t
.
index
[
"company_id"
],
name:
"index_jobs_on_company_id"
end
...
...
@@ -90,11 +90,6 @@ ActiveRecord::Schema.define(version: 2020_03_20_101302) do
add_foreign_key
"applies"
,
"jobs"
add_foreign_key
"applies"
,
"users"
add_foreign_key
"city_jobs"
,
"cities"
add_foreign_key
"city_jobs"
,
"jobs"
add_foreign_key
"favorites"
,
"jobs"
add_foreign_key
"favorites"
,
"users"
add_foreign_key
"industry_jobs"
,
"industries"
add_foreign_key
"industry_jobs"
,
"jobs"
add_foreign_key
"jobs"
,
"companies"
end
lib/tasks/crawler.rake
View file @
64410b1c
...
...
@@ -33,7 +33,7 @@ namespace :crawler do
salary:
nil
,
expiration_date:
nil
,
job_description:
nil
,
company_id:
nil
company_id:
nil
,
}
# Company attributes
...
...
@@ -45,10 +45,10 @@ namespace :crawler do
}
# Defind city ids array
cit
y_id
s
=
[]
cit
ie
s
=
[]
# Defind industry ids array
industr
y_id
s
=
[]
industr
ie
s
=
[]
# Check what template job belongs to
if
html_job_detail
.
at_css
(
"#uni_container .MyJobDetail"
)
...
...
@@ -75,8 +75,8 @@ namespace :crawler do
# Check exist or create city
ele
.
css
(
"b a"
).
each_with_index
do
|
ele
,
index
|
if
index
>
0
city
_id
=
check_exist_or_create_city
(
ele
.
text
.
gsub
(
","
,
""
))
cit
y_ids
<<
city_id
city
=
check_exist_or_create_city
(
ele
.
text
.
gsub
(
","
,
""
))
cit
ies
<<
city
end
end
when
"Cấp bậc: "
...
...
@@ -88,8 +88,8 @@ namespace :crawler do
when
"Ngành nghề: "
# Check exist or create industry
ele
.
css
(
"b a"
).
each_with_index
do
|
ele
,
index
|
industry
_id
=
check_exist_or_create_industry
(
ele
.
text
.
gsub
(
","
,
""
))
industr
y_ids
<<
industry_id
industry
=
check_exist_or_create_industry
(
ele
.
text
.
gsub
(
","
,
""
))
industr
ies
<<
industry
end
else
job_attributes
[
:expiration_date
]
=
ele
.
text
.
gsub
(
"Hết hạn nộp: "
,
""
)
...
...
@@ -131,8 +131,8 @@ namespace :crawler do
# Check exist or create city
ele
.
css
(
"span a"
).
each_with_index
do
|
ele
,
index
|
if
index
>
0
city
_id
=
check_exist_or_create_city
(
ele
.
text
.
gsub
(
","
,
""
))
cit
y_ids
<<
city_id
city
=
check_exist_or_create_city
(
ele
.
text
.
gsub
(
","
,
""
))
cit
ies
<<
city
end
end
when
"Cấp bậc"
...
...
@@ -144,8 +144,8 @@ namespace :crawler do
when
"Ngành nghề"
# Check exist or create industry
ele
.
css
(
"span a"
).
each_with_index
do
|
ele
,
index
|
industry
_id
=
check_exist_or_create_industry
(
ele
.
text
.
gsub
(
","
,
""
))
industr
y_ids
<<
industry_id
industry
=
check_exist_or_create_industry
(
ele
.
text
.
gsub
(
","
,
""
))
industr
ies
<<
industry
end
else
job_attributes
[
:expiration_date
]
=
ele
.
css
(
"span"
).
text
...
...
@@ -179,8 +179,8 @@ namespace :crawler do
# Check exist or create city
ele
.
css
(
"span a"
).
each_with_index
do
|
ele
,
index
|
if
index
>
0
city
_id
=
check_exist_or_create_city
(
ele
.
text
)
cit
y_ids
<<
city_id
city
=
check_exist_or_create_city
(
ele
.
text
)
cit
ies
<<
city
end
end
when
"Cấp bậc"
...
...
@@ -190,8 +190,8 @@ namespace :crawler do
when
"Ngành nghề"
# Check exist or create industry
ele
.
css
(
"span a"
).
each_with_index
do
|
ele
,
index
|
industry
_id
=
check_exist_or_create_industry
(
ele
.
text
)
industr
y_ids
<<
industry_id
industry
=
check_exist_or_create_industry
(
ele
.
text
)
industr
ies
<<
industry
end
when
"Hết hạn nộp"
job_attributes
[
:expiration_date
]
=
ele
.
css
(
"span"
).
text
...
...
@@ -209,21 +209,20 @@ namespace :crawler do
# Check exist or create company
job_attributes
[
:company_id
]
=
check_exist_or_create_company
(
company_attributes
)
# Create job
job
_id
=
check_exist_or_create_job
(
job_attributes
)
job
=
check_exist_or_create_job
(
job_attributes
)
# Create city_job
if
cit
y_id
s
.
length
>
0
cit
y_ids
.
each
do
|
city_id
|
check_exist_or_create_city_job
(
city_id
,
job_id
)
if
cit
ie
s
.
length
>
0
cit
ies
.
each
do
|
city
|
job
.
cities
<<
city
end
end
# Create industry_job
if
industr
y_id
s
.
length
>
0
industr
y_ids
.
each
do
|
industry_id
|
check_exist_or_create_industry_job
(
industry_id
,
job_id
)
if
industr
ie
s
.
length
>
0
industr
ies
.
each
do
|
industry
|
job
.
industries
<<
industry
end
end
...
...
@@ -241,25 +240,16 @@ namespace :crawler do
def
check_exist_or_create_city
(
city_title
)
find_city
=
City
.
find_or_create_by
(
title:
city_title
)
return
find_city
.
id
return
find_city
end
def
check_exist_or_create_industry
(
industry_title
)
find_industry
=
Industry
.
find_or_create_by
(
title:
industry_title
)
return
find_industry
.
id
return
find_industry
end
def
check_exist_or_create_job
(
job_attributes
)
job
=
Job
.
find_or_create_by
(
job_attributes
)
return
job
.
id
end
def
check_exist_or_create_city_job
(
city_id
,
job_id
)
return
CityJob
.
find_or_create_by
(
city_id:
city_id
,
job_id:
job_id
)
return
job
end
def
check_exist_or_create_industry_job
(
industry_id
,
job_id
)
return
IndustryJob
.
find_or_create_by
(
industry_id:
industry_id
,
job_id:
job_id
)
end
end
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment