Skip to content

V
ventura-ghr
Commit 91efbf55 by Bui Minh Duc
Options

implement model Comment, insert data from gitbug

parent 839e5615
Showing with 342 additions and 31 deletions
app/models/comment.rb 0 → 100644
class Comment < ApplicationRecord
belongs_to :issue
belongs_to :user
end
app/models/issue.rb
......@@ -3,4 +3,6 @@ class Issue < ApplicationRecord
belongs_to :user, optional: true #owner
has_and_belongs_to_many :labels
has_and_belongs_to_many :users # assignees
has_many :comments
has_many :review_comments
end
app/models/review_comment.rb 0 → 100644
class ReviewComment < ApplicationRecord
belongs_to :issue
belongs_to :user
end
config/schedule.rb
......@@ -20,6 +20,6 @@
# Learn more: http://github.com/javan/whenever
set :environment, "development"
every 1.hour do
every 2.hour do
rake "github:insert_github_data"
end
db/migrate/20161205090006_create_comments.rb 0 → 100644
class CreateComments < ActiveRecord::Migration[5.0]
def change
create_table :comments do |t|
t.string :html_url
t.text :body
t.references :issue, foreign_key: true
t.references :user, foreign_key: true
t.timestamps
end
end
end
db/migrate/20161206054436_add_closed_at_and_is_pull_to_issue.rb 0 → 100644
class AddClosedAtAndIsPullToIssue < ActiveRecord::Migration[5.0]
def change
add_column :issues, :closed_at, :date
add_column :issues, :is_pull, :boolean
end
end
db/migrate/20161206091833_create_review_comments.rb 0 → 100644
class CreateReviewComments < ActiveRecord::Migration[5.0]
def change
create_table :review_comments do |t|
t.string :url
t.string :diff_hunk
t.string :path
t.string :body
t.string :html_url
t.string :pull_request_url
t.references :issue, foreign_key: true
t.references :user, foreign_key: true
t.timestamps
end
end
end
db/schema.rb
......@@ -10,7 +10,18 @@
#
# It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema.define(version: 20161202062857) do
ActiveRecord::Schema.define(version: 20161206091833) do
create_table "comments", force: :cascade do |t|
t.string "html_url"
t.text "body"
t.integer "issue_id"
t.integer "user_id"
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
t.index ["issue_id"], name: "index_comments_on_issue_id"
t.index ["user_id"], name: "index_comments_on_user_id"
end
create_table "issues", force: :cascade do |t|
t.string "url"
......@@ -24,6 +35,8 @@ ActiveRecord::Schema.define(version: 20161202062857) do
t.datetime "updated_at", null: false
t.string "html_url"
t.boolean "open"
t.date "closed_at"
t.boolean "is_pull"
t.index ["repository_id"], name: "index_issues_on_repository_id"
t.index ["user_id"], name: "index_issues_on_user_id"
end
......@@ -74,6 +87,21 @@ ActiveRecord::Schema.define(version: 20161202062857) do
t.datetime "updated_at", null: false
end
create_table "review_comments", force: :cascade do |t|
t.string "url"
t.string "diff_hunk"
t.string "path"
t.string "body"
t.string "html_url"
t.string "pull_request_url"
t.integer "issue_id"
t.integer "user_id"
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
t.index ["issue_id"], name: "index_review_comments_on_issue_id"
t.index ["user_id"], name: "index_review_comments_on_user_id"
end
create_table "users", force: :cascade do |t|
t.string "login"
t.string "url"
......
design.dia
No preview for this file type
design.png
design.png

8.83 KB | W: | H:

design.png

14.4 KB | W: | H:

design.png
design.png
design.png
design.png
  • 2-up
  • Swipe
  • Onion skin
github.log
......@@ -99,3 +99,93 @@ I, [2016-12-05T14:00:02.715604 #3731] INFO -- : Insert repos
I, [2016-12-05T14:00:08.486465 #3731] INFO -- : Insert users
I, [2016-12-05T14:00:14.200676 #3731] INFO -- : Insert labels
I, [2016-12-05T14:03:32.452579 #3731] INFO -- : Insert issues
I, [2016-12-05T14:07:32.107801 #3731] INFO -- : begin transaction
I, [2016-12-05T14:07:40.543552 #3731] INFO -- : Finished transaction in 8.435613077s
I, [2016-12-05T14:07:40.543680 #3731] INFO -- : Finished task import in 457.828072263s
I, [2016-12-05T15:00:03.133611 #6769] INFO -- : Begin task insert database from github
I, [2016-12-05T15:00:03.133705 #6769] INFO -- : Insert repos
I, [2016-12-05T15:00:08.796443 #6769] INFO -- : Insert users
I, [2016-12-05T15:00:14.600546 #6769] INFO -- : Insert labels
I, [2016-12-05T15:03:33.354420 #6769] INFO -- : Insert issues
I, [2016-12-05T15:07:31.953052 #6769] INFO -- : begin transaction
I, [2016-12-05T15:07:41.011235 #6769] INFO -- : Finished transaction in 9.058078059s
I, [2016-12-05T15:07:41.011319 #6769] INFO -- : Finished task import in 457.877610106s
I, [2016-12-05T16:00:02.762749 #9360] INFO -- : Begin task insert database from github
I, [2016-12-05T16:00:02.762856 #9360] INFO -- : Insert repos
I, [2016-12-05T16:00:08.640470 #9360] INFO -- : Insert users
I, [2016-12-05T16:00:14.985160 #9360] INFO -- : Insert labels
I, [2016-12-05T16:03:39.936936 #9360] INFO -- : Insert issues
I, [2016-12-05T16:07:41.678814 #9360] INFO -- : begin transaction
I, [2016-12-05T16:07:48.789798 #9360] INFO -- : Finished transaction in 7.110761414s
I, [2016-12-05T16:07:48.789939 #9360] INFO -- : Finished task import in 466.027077571s
I, [2016-12-06T13:00:03.362427 #6549] INFO -- : Begin task insert database from github
I, [2016-12-06T13:00:03.375828 #6549] INFO -- : Insert repos
I, [2016-12-06T13:00:08.923237 #6549] INFO -- : Insert users
I, [2016-12-06T13:00:14.673876 #6549] INFO -- : Insert labels
I, [2016-12-06T13:02:51.026603 #6878] INFO -- : Begin task insert database from github
I, [2016-12-06T13:02:51.026699 #6878] INFO -- : Insert repos
I, [2016-12-06T13:02:56.211334 #6878] INFO -- : Insert users
I, [2016-12-06T13:03:01.805284 #6878] INFO -- : Insert labels
I, [2016-12-06T13:03:33.930930 #6549] INFO -- : Insert issues
I, [2016-12-06T13:06:21.957471 #6878] INFO -- : Insert issues
I, [2016-12-06T13:10:29.101017 #6549] INFO -- : begin transaction
I, [2016-12-06T13:10:44.923480 #6549] INFO -- : Finished transaction in 15.822343569s
I, [2016-12-06T13:10:44.923949 #6549] INFO -- : Finished task import in 641.548108686s
I, [2016-12-06T13:13:14.190436 #6878] INFO -- : begin transaction
I, [2016-12-06T13:13:36.306304 #6878] INFO -- : Finished transaction in 22.115765761s
I, [2016-12-06T13:13:36.306386 #6878] INFO -- : Finished task import in 645.279685128s
I, [2016-12-06T13:35:27.938874 #8717] INFO -- : Begin task insert database from github
I, [2016-12-06T13:35:27.938962 #8717] INFO -- : Insert repos
I, [2016-12-06T13:35:33.166934 #8717] INFO -- : Insert users
I, [2016-12-06T13:35:38.907138 #8717] INFO -- : Insert labels
I, [2016-12-06T13:38:58.134040 #8717] INFO -- : Insert issues
I, [2016-12-06T13:54:54.604502 #9828] INFO -- : Begin task insert database from github
I, [2016-12-06T13:54:54.604597 #9828] INFO -- : Insert repos
I, [2016-12-06T13:54:59.613817 #9828] INFO -- : Insert users
I, [2016-12-06T13:55:05.419040 #9828] INFO -- : Insert labels
I, [2016-12-06T13:58:24.807347 #9828] INFO -- : Insert issues
I, [2016-12-06T14:14:00.376827 #12521] INFO -- : Begin task insert database from github
I, [2016-12-06T14:14:57.114514 #12559] INFO -- : Begin task insert database from github
I, [2016-12-06T14:15:22.506051 #12594] INFO -- : Begin task insert database from github
I, [2016-12-06T14:15:24.609571 #12594] INFO -- : Insert comments
I, [2016-12-06T14:16:00.634168 #12674] INFO -- : Begin task insert database from github
I, [2016-12-06T14:16:00.737891 #12674] INFO -- : Insert comments
I, [2016-12-06T14:17:49.648162 #12926] INFO -- : Begin task insert database from github
I, [2016-12-06T14:17:49.750850 #12926] INFO -- : Insert comments
I, [2016-12-06T14:24:48.206410 #13787] INFO -- : Begin task insert database from github
I, [2016-12-06T14:24:48.309407 #13787] INFO -- : Insert comments
I, [2016-12-06T14:39:35.587800 #14358] INFO -- : Begin task insert database from github
I, [2016-12-06T14:39:35.694037 #14358] INFO -- : Insert comments
I, [2016-12-06T14:40:10.204060 #14410] INFO -- : Begin task insert database from github
I, [2016-12-06T14:40:10.328464 #14410] INFO -- : Insert comments
I, [2016-12-06T14:40:38.694438 #14456] INFO -- : Begin task insert database from github
I, [2016-12-06T14:40:38.793812 #14456] INFO -- : Insert comments
I, [2016-12-06T14:41:15.428142 #14498] INFO -- : Begin task insert database from github
I, [2016-12-06T14:41:15.526626 #14498] INFO -- : Insert comments
I, [2016-12-06T14:41:44.672500 #14524] INFO -- : Begin task insert database from github
I, [2016-12-06T14:41:44.770497 #14524] INFO -- : Insert comments
I, [2016-12-06T14:53:55.467665 #15071] INFO -- : Begin task insert database from github
I, [2016-12-06T14:53:55.566455 #15071] INFO -- : Insert comments
I, [2016-12-06T14:59:13.426295 #15071] INFO -- : begin transaction
I, [2016-12-06T14:59:18.281372 #15071] INFO -- : Finished transaction in 4.854989146s
I, [2016-12-06T14:59:18.281432 #15071] INFO -- : Finished task import in 322.813591551s
I, [2016-12-06T16:39:14.682823 #18785] INFO -- : Begin task insert database from github
I, [2016-12-06T16:39:14.783855 #18785] INFO -- : begin transaction
I, [2016-12-06T16:40:11.418342 #18832] INFO -- : Begin task insert database from github
I, [2016-12-06T16:40:11.516399 #18832] INFO -- : Insert review comments
I, [2016-12-06T16:40:49.932572 #18875] INFO -- : Begin task insert database from github
I, [2016-12-06T16:40:50.032380 #18875] INFO -- : Insert review comments
I, [2016-12-06T16:47:01.030424 #18875] INFO -- : begin transaction
I, [2016-12-06T16:47:03.662509 #18875] INFO -- : Finished transaction in 2.631867556s
I, [2016-12-06T16:47:03.662575 #18875] INFO -- : Finished task import in 373.729898439s
I, [2016-12-06T16:55:08.675460 #19686] INFO -- : Begin task insert database from github
I, [2016-12-06T16:55:08.775452 #19686] INFO -- : Insert review comments
I, [2016-12-06T17:00:59.606493 #19686] INFO -- : begin transaction
I, [2016-12-06T17:07:40.999281 #20746] INFO -- : Begin task insert database from github
I, [2016-12-06T17:07:41.099472 #20746] INFO -- : Insert review comments
I, [2016-12-06T17:08:24.257306 #20788] INFO -- : Begin task insert database from github
I, [2016-12-06T17:08:24.355684 #20788] INFO -- : Insert review comments
I, [2016-12-06T17:09:26.069596 #20842] INFO -- : Begin task insert database from github
I, [2016-12-06T17:09:26.167977 #20842] INFO -- : Insert review comments
I, [2016-12-06T17:11:43.931030 #20946] INFO -- : Begin task insert database from github
I, [2016-12-06T17:11:44.029829 #20946] INFO -- : Insert review comments
lib/tasks/github.rake
......@@ -12,34 +12,40 @@ namespace :github do
task insert_github_data: :environment do
$logger.info "Begin task insert database from github"
start_time = Time.now
list_db_repo = insert_repos($client)
list_db_user = insert_users($client)
list_db_label = insert_labels($client)
list_db_issue = insert_issues($client, list_db_user, list_db_label)
# list_db_repo = insert_repos($client)
# list_db_user = insert_users($client)
# list_db_label = insert_labels($client, list_db_repo)
# list_db_issue = insert_issues($client, list_db_user, list_db_label, list_db_repo)
list_db_repo = Repository.all
# list_db_comment = insert_comments($client, list_db_repo)
list_db_rv_comment = insert_review_comments($client, list_db_repo)
$logger.info "begin transaction"
start_time_transaction = Time.now
ActiveRecord::Base.transaction do
Repository.destroy_all
User.destroy_all
Label.destroy_all
Issue.destroy_all
# list_db_repo.each do |db_repo|
# db_repo.save
# end
list_db_repo.each do |db_repo|
db_repo.save
end
# list_db_user.each do |db_user|
# db_user.save
# end
list_db_user.each do |db_user|
db_user.save
end
# list_db_label.each do |db_label|
# db_label.save
# end
list_db_label.each do |db_label|
db_label.save
end
# list_db_issue.each do |db_issue|
# db_issue.save
# end
# list_db_comment.each do |db_comment|
# db_comment.save
# end
list_db_issue.each do |db_issue|
db_issue.save
list_db_rv_comment.each do |db_rv_comment|
db_rv_comment.save
end
end
$logger.info "Finished transaction in #{Time.now - start_time_transaction}s"
......@@ -54,10 +60,12 @@ namespace :github do
last_page = client.last_response.headers[:link].split(' ')[2][-3].to_i
end
list_db_repo = []
list_db_repo = Repository.all
for i in (1..last_page)
repos = client.org_repos($org, {type: 'all', page: i})
repos.each do |repo|
db_repo = Repository.find_by(id: repo.id)
if db_repo.nil?
db_repo = Repository.new
db_repo.id = repo.id
db_repo.name = repo.name
......@@ -72,12 +80,12 @@ namespace :github do
list_db_repo.append(db_repo)
end
end
end
list_db_repo
end
def insert_labels(client)
def insert_labels(client, db_repos)
$logger.info "Insert labels"
db_repos = Repository.all
list_db_label = []
db_repos.each do |repo|
labels = client.labels($org + "/" + repo.name)
......@@ -89,10 +97,13 @@ namespace :github do
for i in (1..last_page)
labels = client.labels($org + "/" + repo.name, {page: i})
labels.each do |label|
db_label = Label.find_by(id: label.id)
if db_label.nil?
db_label = Label.new
db_label.id = label.id
db_label.url = label.url
db_label.repository_id = repo.id
end
db_label.name = label.name
db_label.color = label.color
db_label.default = label.default
......@@ -112,10 +123,12 @@ namespace :github do
last_page = client.last_response.headers[:link].split(' ')[2][-3].to_i
end
list_db_user = []
list_db_user = User.all
for i in (1..last_page)
users = client.org_members($org, {page: i})
users.each do |user|
db_user = User.find_by(id: user.id)
if db_user.nil?
db_user = User.new
db_user.id = user.id
db_user.login = user.login
......@@ -125,24 +138,26 @@ namespace :github do
list_db_user.append(db_user)
end
end
end
list_db_user
end
def insert_issues(client, list_db_user, list_db_label)
def insert_issues(client, list_db_user, list_db_label, db_repos)
$logger.info "Insert issues"
db_repos = Repository.all
list_db_issue = []
db_repos.each do |repo|
issues = client.list_issues($org + "/" + repo.name)
issues = client.list_issues($org + "/" + repo.name, { state: "all" })
last_page = 1
unless client.last_response.headers[:link].nil?
last_page = client.last_response.headers[:link].split(' ')[2][-3].to_i
end
for i in (1..last_page)
issues = client.list_issues($org + "/" + repo.name, {page: i})
issues = client.list_issues($org + "/" + repo.name, { state: "all", page: i })
issues.each do |issue|
db_issue = Issue.find_by(id: issue.id)
if db_issue.nil?
db_issue = Issue.new
db_issue.id = issue.id
db_issue.url = issue.url
......@@ -150,12 +165,19 @@ namespace :github do
db_issue.number = issue.number
db_issue.repository_id = repo.id
db_issue.user_id = issue.user.id
db_issue.created_at = issue.created_at
db_issue.html_url = issue.html_url
if issue[:pull_request].nil?
db_issue.is_pull = false
else
db_issue.is_pull = true
end
end
db_issue.title = issue.title
db_issue.body = issue.body
db_issue.html_url = issue.html_url
db_issue.created_at = issue.created_at
db_issue.updated_at = issue.updated_at
db_issue.closed_at = issue.closed_at
db_issue.labels.clear
issue.labels.each do |label|
......@@ -184,4 +206,83 @@ namespace :github do
list_db_issue
end
def insert_comments(client, list_db_repo)
$logger.info "Insert comments"
list_db_comment = []
list_db_repo.each do |db_repo|
comments = client.issues_comments($org + "/" + db_repo.name)
last_page = 1
unless client.last_response.headers[:link].nil?
last_page = client.last_response.headers[:link].split(' ')[2][-3].to_i
end
for i in (1..last_page)
comments = client.issues_comments($org + "/" + db_repo.name, { page: i })
comments.each do |comment|
db_comment = Comment.find_by(id: comment.id)
if db_comment.nil?
db_comment = Comment.new
db_comment.id = comment.id
db_comment.html_url = comment.html_url
issue_url = comment[:issue_url].split("/")
issue = Repository.find_by(name: issue_url[-3]).issues.find_by(number: issue_url[-1])
unless issue.nil?
db_comment.issue_id = issue.id
end
db_comment.user_id = comment.user.id
end
db_comment.body = comment.body
list_db_comment.append(db_comment)
end
end
end
$logger.info "Size list_db_comment: " + list_db_comment.size.to_s
list_db_comment
end
def insert_review_comments(client, list_db_repo)
$logger.info "Insert review comments"
list_db_rv_comment = []
list_db_repo.each do |db_repo|
rv_comments = client.pull_requests_comments($org + "/" + db_repo.name)
last_page = 1
unless client.last_response.headers[:link].nil?
last_page = client.last_response.headers[:link].split(' ')[2][-3].to_i
end
for i in (1..last_page)
rv_comments = client.pull_requests_comments($org + "/" + db_repo.name, { page: i })
rv_comments.each do |rv_comment|
db_rv_comment = ReviewComment.find_by(id: rv_comment.id)
if db_rv_comment.nil?
db_rv_comment = ReviewComment.new
db_rv_comment.id = rv_comment.id
db_rv_comment.url = rv_comment.url
db_rv_comment.diff_hunk = rv_comment.diff_hunk
db_rv_comment.path = rv_comment.path
db_rv_comment.html_url = rv_comment.html_url
db_rv_comment.pull_request_url = rv_comment.pull_request_url
db_rv_comment.created_at = rv_comment.created_at
unless rv_comment.user.nil?
db_rv_comment.user_id = rv_comment.user.id
end
pull_request_url_split = rv_comment.pull_request_url.split("/")
issue = Repository.find_by(name: pull_request_url_split[-3]).issues.find_by(number: pull_request_url_split[-1])
unless issue.nil?
db_rv_comment.issue_id = issue.id
end
end
db_rv_comment.updated_at = rv_comment.updated_at
db_rv_comment.body = rv_comment.body
list_db_rv_comment.append(db_rv_comment)
end
end
end
list_db_rv_comment
end
end
test/fixtures/comments.yml 0 → 100644
# Read about fixtures at http://api.rubyonrails.org/classes/ActiveRecord/FixtureSet.html
one:
html_url: MyString
body: MyText
issue: one
user: one
two:
html_url: MyString
body: MyText
issue: two
user: two
test/fixtures/review_comments.yml 0 → 100644
# Read about fixtures at http://api.rubyonrails.org/classes/ActiveRecord/FixtureSet.html
one:
url: MyString
diff_hunk: MyString
path: MyString
body: MyString
html_url: MyString
pull_request_url: MyString
issue: one
user: one
two:
url: MyString
diff_hunk: MyString
path: MyString
body: MyString
html_url: MyString
pull_request_url: MyString
issue: two
user: two
test/models/comment_test.rb 0 → 100644
require 'test_helper'
class CommentTest < ActiveSupport::TestCase
# test "the truth" do
# assert true
# end
end
test/models/review_comment_test.rb 0 → 100644
require 'test_helper'
class ReviewCommentTest < ActiveSupport::TestCase
# test "the truth" do
# assert true
# end
end
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment