Commit 91efbf55 by Bui Minh Duc

implement model Comment, insert data from gitbug

parent 839e5615
class Comment < ApplicationRecord
belongs_to :issue
belongs_to :user
end
...@@ -3,4 +3,6 @@ class Issue < ApplicationRecord ...@@ -3,4 +3,6 @@ class Issue < ApplicationRecord
belongs_to :user, optional: true #owner belongs_to :user, optional: true #owner
has_and_belongs_to_many :labels has_and_belongs_to_many :labels
has_and_belongs_to_many :users # assignees has_and_belongs_to_many :users # assignees
has_many :comments
has_many :review_comments
end end
class ReviewComment < ApplicationRecord
belongs_to :issue
belongs_to :user
end
...@@ -20,6 +20,6 @@ ...@@ -20,6 +20,6 @@
# Learn more: http://github.com/javan/whenever # Learn more: http://github.com/javan/whenever
set :environment, "development" set :environment, "development"
every 1.hour do every 2.hour do
rake "github:insert_github_data" rake "github:insert_github_data"
end end
class CreateComments < ActiveRecord::Migration[5.0]
def change
create_table :comments do |t|
t.string :html_url
t.text :body
t.references :issue, foreign_key: true
t.references :user, foreign_key: true
t.timestamps
end
end
end
class AddClosedAtAndIsPullToIssue < ActiveRecord::Migration[5.0]
def change
add_column :issues, :closed_at, :date
add_column :issues, :is_pull, :boolean
end
end
class CreateReviewComments < ActiveRecord::Migration[5.0]
def change
create_table :review_comments do |t|
t.string :url
t.string :diff_hunk
t.string :path
t.string :body
t.string :html_url
t.string :pull_request_url
t.references :issue, foreign_key: true
t.references :user, foreign_key: true
t.timestamps
end
end
end
...@@ -10,7 +10,18 @@ ...@@ -10,7 +10,18 @@
# #
# It's strongly recommended that you check this file into your version control system. # It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema.define(version: 20161202062857) do ActiveRecord::Schema.define(version: 20161206091833) do
create_table "comments", force: :cascade do |t|
t.string "html_url"
t.text "body"
t.integer "issue_id"
t.integer "user_id"
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
t.index ["issue_id"], name: "index_comments_on_issue_id"
t.index ["user_id"], name: "index_comments_on_user_id"
end
create_table "issues", force: :cascade do |t| create_table "issues", force: :cascade do |t|
t.string "url" t.string "url"
...@@ -24,6 +35,8 @@ ActiveRecord::Schema.define(version: 20161202062857) do ...@@ -24,6 +35,8 @@ ActiveRecord::Schema.define(version: 20161202062857) do
t.datetime "updated_at", null: false t.datetime "updated_at", null: false
t.string "html_url" t.string "html_url"
t.boolean "open" t.boolean "open"
t.date "closed_at"
t.boolean "is_pull"
t.index ["repository_id"], name: "index_issues_on_repository_id" t.index ["repository_id"], name: "index_issues_on_repository_id"
t.index ["user_id"], name: "index_issues_on_user_id" t.index ["user_id"], name: "index_issues_on_user_id"
end end
...@@ -74,6 +87,21 @@ ActiveRecord::Schema.define(version: 20161202062857) do ...@@ -74,6 +87,21 @@ ActiveRecord::Schema.define(version: 20161202062857) do
t.datetime "updated_at", null: false t.datetime "updated_at", null: false
end end
create_table "review_comments", force: :cascade do |t|
t.string "url"
t.string "diff_hunk"
t.string "path"
t.string "body"
t.string "html_url"
t.string "pull_request_url"
t.integer "issue_id"
t.integer "user_id"
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
t.index ["issue_id"], name: "index_review_comments_on_issue_id"
t.index ["user_id"], name: "index_review_comments_on_user_id"
end
create_table "users", force: :cascade do |t| create_table "users", force: :cascade do |t|
t.string "login" t.string "login"
t.string "url" t.string "url"
......
No preview for this file type

8.83 KB | W: | H:

14.4 KB | W: | H:

design.png
design.png
design.png
design.png
  • 2-up
  • Swipe
  • Onion skin
...@@ -99,3 +99,93 @@ I, [2016-12-05T14:00:02.715604 #3731] INFO -- : Insert repos ...@@ -99,3 +99,93 @@ I, [2016-12-05T14:00:02.715604 #3731] INFO -- : Insert repos
I, [2016-12-05T14:00:08.486465 #3731] INFO -- : Insert users I, [2016-12-05T14:00:08.486465 #3731] INFO -- : Insert users
I, [2016-12-05T14:00:14.200676 #3731] INFO -- : Insert labels I, [2016-12-05T14:00:14.200676 #3731] INFO -- : Insert labels
I, [2016-12-05T14:03:32.452579 #3731] INFO -- : Insert issues I, [2016-12-05T14:03:32.452579 #3731] INFO -- : Insert issues
I, [2016-12-05T14:07:32.107801 #3731] INFO -- : begin transaction
I, [2016-12-05T14:07:40.543552 #3731] INFO -- : Finished transaction in 8.435613077s
I, [2016-12-05T14:07:40.543680 #3731] INFO -- : Finished task import in 457.828072263s
I, [2016-12-05T15:00:03.133611 #6769] INFO -- : Begin task insert database from github
I, [2016-12-05T15:00:03.133705 #6769] INFO -- : Insert repos
I, [2016-12-05T15:00:08.796443 #6769] INFO -- : Insert users
I, [2016-12-05T15:00:14.600546 #6769] INFO -- : Insert labels
I, [2016-12-05T15:03:33.354420 #6769] INFO -- : Insert issues
I, [2016-12-05T15:07:31.953052 #6769] INFO -- : begin transaction
I, [2016-12-05T15:07:41.011235 #6769] INFO -- : Finished transaction in 9.058078059s
I, [2016-12-05T15:07:41.011319 #6769] INFO -- : Finished task import in 457.877610106s
I, [2016-12-05T16:00:02.762749 #9360] INFO -- : Begin task insert database from github
I, [2016-12-05T16:00:02.762856 #9360] INFO -- : Insert repos
I, [2016-12-05T16:00:08.640470 #9360] INFO -- : Insert users
I, [2016-12-05T16:00:14.985160 #9360] INFO -- : Insert labels
I, [2016-12-05T16:03:39.936936 #9360] INFO -- : Insert issues
I, [2016-12-05T16:07:41.678814 #9360] INFO -- : begin transaction
I, [2016-12-05T16:07:48.789798 #9360] INFO -- : Finished transaction in 7.110761414s
I, [2016-12-05T16:07:48.789939 #9360] INFO -- : Finished task import in 466.027077571s
I, [2016-12-06T13:00:03.362427 #6549] INFO -- : Begin task insert database from github
I, [2016-12-06T13:00:03.375828 #6549] INFO -- : Insert repos
I, [2016-12-06T13:00:08.923237 #6549] INFO -- : Insert users
I, [2016-12-06T13:00:14.673876 #6549] INFO -- : Insert labels
I, [2016-12-06T13:02:51.026603 #6878] INFO -- : Begin task insert database from github
I, [2016-12-06T13:02:51.026699 #6878] INFO -- : Insert repos
I, [2016-12-06T13:02:56.211334 #6878] INFO -- : Insert users
I, [2016-12-06T13:03:01.805284 #6878] INFO -- : Insert labels
I, [2016-12-06T13:03:33.930930 #6549] INFO -- : Insert issues
I, [2016-12-06T13:06:21.957471 #6878] INFO -- : Insert issues
I, [2016-12-06T13:10:29.101017 #6549] INFO -- : begin transaction
I, [2016-12-06T13:10:44.923480 #6549] INFO -- : Finished transaction in 15.822343569s
I, [2016-12-06T13:10:44.923949 #6549] INFO -- : Finished task import in 641.548108686s
I, [2016-12-06T13:13:14.190436 #6878] INFO -- : begin transaction
I, [2016-12-06T13:13:36.306304 #6878] INFO -- : Finished transaction in 22.115765761s
I, [2016-12-06T13:13:36.306386 #6878] INFO -- : Finished task import in 645.279685128s
I, [2016-12-06T13:35:27.938874 #8717] INFO -- : Begin task insert database from github
I, [2016-12-06T13:35:27.938962 #8717] INFO -- : Insert repos
I, [2016-12-06T13:35:33.166934 #8717] INFO -- : Insert users
I, [2016-12-06T13:35:38.907138 #8717] INFO -- : Insert labels
I, [2016-12-06T13:38:58.134040 #8717] INFO -- : Insert issues
I, [2016-12-06T13:54:54.604502 #9828] INFO -- : Begin task insert database from github
I, [2016-12-06T13:54:54.604597 #9828] INFO -- : Insert repos
I, [2016-12-06T13:54:59.613817 #9828] INFO -- : Insert users
I, [2016-12-06T13:55:05.419040 #9828] INFO -- : Insert labels
I, [2016-12-06T13:58:24.807347 #9828] INFO -- : Insert issues
I, [2016-12-06T14:14:00.376827 #12521] INFO -- : Begin task insert database from github
I, [2016-12-06T14:14:57.114514 #12559] INFO -- : Begin task insert database from github
I, [2016-12-06T14:15:22.506051 #12594] INFO -- : Begin task insert database from github
I, [2016-12-06T14:15:24.609571 #12594] INFO -- : Insert comments
I, [2016-12-06T14:16:00.634168 #12674] INFO -- : Begin task insert database from github
I, [2016-12-06T14:16:00.737891 #12674] INFO -- : Insert comments
I, [2016-12-06T14:17:49.648162 #12926] INFO -- : Begin task insert database from github
I, [2016-12-06T14:17:49.750850 #12926] INFO -- : Insert comments
I, [2016-12-06T14:24:48.206410 #13787] INFO -- : Begin task insert database from github
I, [2016-12-06T14:24:48.309407 #13787] INFO -- : Insert comments
I, [2016-12-06T14:39:35.587800 #14358] INFO -- : Begin task insert database from github
I, [2016-12-06T14:39:35.694037 #14358] INFO -- : Insert comments
I, [2016-12-06T14:40:10.204060 #14410] INFO -- : Begin task insert database from github
I, [2016-12-06T14:40:10.328464 #14410] INFO -- : Insert comments
I, [2016-12-06T14:40:38.694438 #14456] INFO -- : Begin task insert database from github
I, [2016-12-06T14:40:38.793812 #14456] INFO -- : Insert comments
I, [2016-12-06T14:41:15.428142 #14498] INFO -- : Begin task insert database from github
I, [2016-12-06T14:41:15.526626 #14498] INFO -- : Insert comments
I, [2016-12-06T14:41:44.672500 #14524] INFO -- : Begin task insert database from github
I, [2016-12-06T14:41:44.770497 #14524] INFO -- : Insert comments
I, [2016-12-06T14:53:55.467665 #15071] INFO -- : Begin task insert database from github
I, [2016-12-06T14:53:55.566455 #15071] INFO -- : Insert comments
I, [2016-12-06T14:59:13.426295 #15071] INFO -- : begin transaction
I, [2016-12-06T14:59:18.281372 #15071] INFO -- : Finished transaction in 4.854989146s
I, [2016-12-06T14:59:18.281432 #15071] INFO -- : Finished task import in 322.813591551s
I, [2016-12-06T16:39:14.682823 #18785] INFO -- : Begin task insert database from github
I, [2016-12-06T16:39:14.783855 #18785] INFO -- : begin transaction
I, [2016-12-06T16:40:11.418342 #18832] INFO -- : Begin task insert database from github
I, [2016-12-06T16:40:11.516399 #18832] INFO -- : Insert review comments
I, [2016-12-06T16:40:49.932572 #18875] INFO -- : Begin task insert database from github
I, [2016-12-06T16:40:50.032380 #18875] INFO -- : Insert review comments
I, [2016-12-06T16:47:01.030424 #18875] INFO -- : begin transaction
I, [2016-12-06T16:47:03.662509 #18875] INFO -- : Finished transaction in 2.631867556s
I, [2016-12-06T16:47:03.662575 #18875] INFO -- : Finished task import in 373.729898439s
I, [2016-12-06T16:55:08.675460 #19686] INFO -- : Begin task insert database from github
I, [2016-12-06T16:55:08.775452 #19686] INFO -- : Insert review comments
I, [2016-12-06T17:00:59.606493 #19686] INFO -- : begin transaction
I, [2016-12-06T17:07:40.999281 #20746] INFO -- : Begin task insert database from github
I, [2016-12-06T17:07:41.099472 #20746] INFO -- : Insert review comments
I, [2016-12-06T17:08:24.257306 #20788] INFO -- : Begin task insert database from github
I, [2016-12-06T17:08:24.355684 #20788] INFO -- : Insert review comments
I, [2016-12-06T17:09:26.069596 #20842] INFO -- : Begin task insert database from github
I, [2016-12-06T17:09:26.167977 #20842] INFO -- : Insert review comments
I, [2016-12-06T17:11:43.931030 #20946] INFO -- : Begin task insert database from github
I, [2016-12-06T17:11:44.029829 #20946] INFO -- : Insert review comments
...@@ -12,34 +12,40 @@ namespace :github do ...@@ -12,34 +12,40 @@ namespace :github do
task insert_github_data: :environment do task insert_github_data: :environment do
$logger.info "Begin task insert database from github" $logger.info "Begin task insert database from github"
start_time = Time.now start_time = Time.now
list_db_repo = insert_repos($client) # list_db_repo = insert_repos($client)
list_db_user = insert_users($client) # list_db_user = insert_users($client)
list_db_label = insert_labels($client) # list_db_label = insert_labels($client, list_db_repo)
list_db_issue = insert_issues($client, list_db_user, list_db_label) # list_db_issue = insert_issues($client, list_db_user, list_db_label, list_db_repo)
list_db_repo = Repository.all
# list_db_comment = insert_comments($client, list_db_repo)
list_db_rv_comment = insert_review_comments($client, list_db_repo)
$logger.info "begin transaction" $logger.info "begin transaction"
start_time_transaction = Time.now start_time_transaction = Time.now
ActiveRecord::Base.transaction do ActiveRecord::Base.transaction do
Repository.destroy_all # list_db_repo.each do |db_repo|
User.destroy_all # db_repo.save
Label.destroy_all # end
Issue.destroy_all
list_db_repo.each do |db_repo| # list_db_user.each do |db_user|
db_repo.save # db_user.save
end # end
list_db_user.each do |db_user| # list_db_label.each do |db_label|
db_user.save # db_label.save
end # end
list_db_label.each do |db_label| # list_db_issue.each do |db_issue|
db_label.save # db_issue.save
end # end
list_db_issue.each do |db_issue| # list_db_comment.each do |db_comment|
db_issue.save # db_comment.save
# end
list_db_rv_comment.each do |db_rv_comment|
db_rv_comment.save
end end
end end
$logger.info "Finished transaction in #{Time.now - start_time_transaction}s" $logger.info "Finished transaction in #{Time.now - start_time_transaction}s"
...@@ -54,30 +60,32 @@ namespace :github do ...@@ -54,30 +60,32 @@ namespace :github do
last_page = client.last_response.headers[:link].split(' ')[2][-3].to_i last_page = client.last_response.headers[:link].split(' ')[2][-3].to_i
end end
list_db_repo = [] list_db_repo = Repository.all
for i in (1..last_page) for i in (1..last_page)
repos = client.org_repos($org, {type: 'all', page: i}) repos = client.org_repos($org, {type: 'all', page: i})
repos.each do |repo| repos.each do |repo|
db_repo = Repository.new db_repo = Repository.find_by(id: repo.id)
db_repo.id = repo.id if db_repo.nil?
db_repo.name = repo.name db_repo = Repository.new
db_repo.full_name = repo.full_name db_repo.id = repo.id
db_repo.private = repo.private db_repo.name = repo.name
db_repo.description = repo.description db_repo.full_name = repo.full_name
db_repo.url = repo.url db_repo.private = repo.private
db_repo.html_url = repo.html_url db_repo.description = repo.description
db_repo.created_at = repo.created_at db_repo.url = repo.url
db_repo.updated_at = repo.updated_at db_repo.html_url = repo.html_url
db_repo.created_at = repo.created_at
list_db_repo.append(db_repo) db_repo.updated_at = repo.updated_at
list_db_repo.append(db_repo)
end
end end
end end
list_db_repo list_db_repo
end end
def insert_labels(client) def insert_labels(client, db_repos)
$logger.info "Insert labels" $logger.info "Insert labels"
db_repos = Repository.all
list_db_label = [] list_db_label = []
db_repos.each do |repo| db_repos.each do |repo|
labels = client.labels($org + "/" + repo.name) labels = client.labels($org + "/" + repo.name)
...@@ -89,10 +97,13 @@ namespace :github do ...@@ -89,10 +97,13 @@ namespace :github do
for i in (1..last_page) for i in (1..last_page)
labels = client.labels($org + "/" + repo.name, {page: i}) labels = client.labels($org + "/" + repo.name, {page: i})
labels.each do |label| labels.each do |label|
db_label = Label.new db_label = Label.find_by(id: label.id)
db_label.id = label.id if db_label.nil?
db_label.url = label.url db_label = Label.new
db_label.repository_id = repo.id db_label.id = label.id
db_label.url = label.url
db_label.repository_id = repo.id
end
db_label.name = label.name db_label.name = label.name
db_label.color = label.color db_label.color = label.color
db_label.default = label.default db_label.default = label.default
...@@ -112,50 +123,61 @@ namespace :github do ...@@ -112,50 +123,61 @@ namespace :github do
last_page = client.last_response.headers[:link].split(' ')[2][-3].to_i last_page = client.last_response.headers[:link].split(' ')[2][-3].to_i
end end
list_db_user = [] list_db_user = User.all
for i in (1..last_page) for i in (1..last_page)
users = client.org_members($org, {page: i}) users = client.org_members($org, {page: i})
users.each do |user| users.each do |user|
db_user = User.new db_user = User.find_by(id: user.id)
db_user.id = user.id if db_user.nil?
db_user.login = user.login db_user = User.new
db_user.url = user.url db_user.id = user.id
db_user.html_url = user.html_url db_user.login = user.login
db_user.url = user.url
db_user.html_url = user.html_url
list_db_user.append(db_user) list_db_user.append(db_user)
end
end end
end end
list_db_user list_db_user
end end
def insert_issues(client, list_db_user, list_db_label) def insert_issues(client, list_db_user, list_db_label, db_repos)
$logger.info "Insert issues" $logger.info "Insert issues"
db_repos = Repository.all
list_db_issue = [] list_db_issue = []
db_repos.each do |repo| db_repos.each do |repo|
issues = client.list_issues($org + "/" + repo.name) issues = client.list_issues($org + "/" + repo.name, { state: "all" })
last_page = 1 last_page = 1
unless client.last_response.headers[:link].nil? unless client.last_response.headers[:link].nil?
last_page = client.last_response.headers[:link].split(' ')[2][-3].to_i last_page = client.last_response.headers[:link].split(' ')[2][-3].to_i
end end
for i in (1..last_page) for i in (1..last_page)
issues = client.list_issues($org + "/" + repo.name, {page: i}) issues = client.list_issues($org + "/" + repo.name, { state: "all", page: i })
issues.each do |issue| issues.each do |issue|
db_issue = Issue.new db_issue = Issue.find_by(id: issue.id)
db_issue.id = issue.id if db_issue.nil?
db_issue.url = issue.url db_issue = Issue.new
db_issue.repository_url = issue.repository_url db_issue.id = issue.id
db_issue.number = issue.number db_issue.url = issue.url
db_issue.repository_id = repo.id db_issue.repository_url = issue.repository_url
db_issue.user_id = issue.user.id db_issue.number = issue.number
db_issue.repository_id = repo.id
db_issue.user_id = issue.user.id
db_issue.created_at = issue.created_at
db_issue.html_url = issue.html_url
if issue[:pull_request].nil?
db_issue.is_pull = false
else
db_issue.is_pull = true
end
end
db_issue.title = issue.title db_issue.title = issue.title
db_issue.body = issue.body db_issue.body = issue.body
db_issue.html_url = issue.html_url
db_issue.created_at = issue.created_at
db_issue.updated_at = issue.updated_at db_issue.updated_at = issue.updated_at
db_issue.closed_at = issue.closed_at
db_issue.labels.clear db_issue.labels.clear
issue.labels.each do |label| issue.labels.each do |label|
...@@ -184,4 +206,83 @@ namespace :github do ...@@ -184,4 +206,83 @@ namespace :github do
list_db_issue list_db_issue
end end
def insert_comments(client, list_db_repo)
$logger.info "Insert comments"
list_db_comment = []
list_db_repo.each do |db_repo|
comments = client.issues_comments($org + "/" + db_repo.name)
last_page = 1
unless client.last_response.headers[:link].nil?
last_page = client.last_response.headers[:link].split(' ')[2][-3].to_i
end
for i in (1..last_page)
comments = client.issues_comments($org + "/" + db_repo.name, { page: i })
comments.each do |comment|
db_comment = Comment.find_by(id: comment.id)
if db_comment.nil?
db_comment = Comment.new
db_comment.id = comment.id
db_comment.html_url = comment.html_url
issue_url = comment[:issue_url].split("/")
issue = Repository.find_by(name: issue_url[-3]).issues.find_by(number: issue_url[-1])
unless issue.nil?
db_comment.issue_id = issue.id
end
db_comment.user_id = comment.user.id
end
db_comment.body = comment.body
list_db_comment.append(db_comment)
end
end
end
$logger.info "Size list_db_comment: " + list_db_comment.size.to_s
list_db_comment
end
def insert_review_comments(client, list_db_repo)
$logger.info "Insert review comments"
list_db_rv_comment = []
list_db_repo.each do |db_repo|
rv_comments = client.pull_requests_comments($org + "/" + db_repo.name)
last_page = 1
unless client.last_response.headers[:link].nil?
last_page = client.last_response.headers[:link].split(' ')[2][-3].to_i
end
for i in (1..last_page)
rv_comments = client.pull_requests_comments($org + "/" + db_repo.name, { page: i })
rv_comments.each do |rv_comment|
db_rv_comment = ReviewComment.find_by(id: rv_comment.id)
if db_rv_comment.nil?
db_rv_comment = ReviewComment.new
db_rv_comment.id = rv_comment.id
db_rv_comment.url = rv_comment.url
db_rv_comment.diff_hunk = rv_comment.diff_hunk
db_rv_comment.path = rv_comment.path
db_rv_comment.html_url = rv_comment.html_url
db_rv_comment.pull_request_url = rv_comment.pull_request_url
db_rv_comment.created_at = rv_comment.created_at
unless rv_comment.user.nil?
db_rv_comment.user_id = rv_comment.user.id
end
pull_request_url_split = rv_comment.pull_request_url.split("/")
issue = Repository.find_by(name: pull_request_url_split[-3]).issues.find_by(number: pull_request_url_split[-1])
unless issue.nil?
db_rv_comment.issue_id = issue.id
end
end
db_rv_comment.updated_at = rv_comment.updated_at
db_rv_comment.body = rv_comment.body
list_db_rv_comment.append(db_rv_comment)
end
end
end
list_db_rv_comment
end
end end
# Read about fixtures at http://api.rubyonrails.org/classes/ActiveRecord/FixtureSet.html
one:
html_url: MyString
body: MyText
issue: one
user: one
two:
html_url: MyString
body: MyText
issue: two
user: two
# Read about fixtures at http://api.rubyonrails.org/classes/ActiveRecord/FixtureSet.html
one:
url: MyString
diff_hunk: MyString
path: MyString
body: MyString
html_url: MyString
pull_request_url: MyString
issue: one
user: one
two:
url: MyString
diff_hunk: MyString
path: MyString
body: MyString
html_url: MyString
pull_request_url: MyString
issue: two
user: two
require 'test_helper'
class CommentTest < ActiveSupport::TestCase
# test "the truth" do
# assert true
# end
end
require 'test_helper'
class ReviewCommentTest < ActiveSupport::TestCase
# test "the truth" do
# assert true
# end
end
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment