Skip to content

V
ventura-ghr
Commit 4f7c94b1 by Bui Minh Duc
Options

migrate to mysql

parent 4e21255b
Showing with 354 additions and 223 deletions
Gemfile
...@@ -29,6 +29,7 @@ gem 'jbuilder', '~> 2.5' ...@@ -29,6 +29,7 @@ gem 'jbuilder', '~> 2.5'
# Use Capistrano for deployment # Use Capistrano for deployment
# gem 'capistrano-rails', group: :development # gem 'capistrano-rails', group: :development
gem 'mysql2', '~> 0.4.5'
gem 'bootstrap-sass', '~> 3.3.6' gem 'bootstrap-sass', '~> 3.3.6'
gem 'octokit', '~> 4.6', '>= 4.6.2' gem 'octokit', '~> 4.6', '>= 4.6.2'
......
Gemfile.lock
...@@ -96,6 +96,7 @@ GEM ...@@ -96,6 +96,7 @@ GEM
railties (>= 3.1) railties (>= 3.1)
multi_json (1.12.1) multi_json (1.12.1)
multipart-post (2.0.0) multipart-post (2.0.0)
mysql2 (0.4.5)
nio4r (1.2.1) nio4r (1.2.1)
nokogiri (1.7.0) nokogiri (1.7.0)
mini_portile2 (~> 2.1.0) mini_portile2 (~> 2.1.0)
...@@ -199,6 +200,7 @@ DEPENDENCIES ...@@ -199,6 +200,7 @@ DEPENDENCIES
jquery-rails jquery-rails
listen (~> 3.0.5) listen (~> 3.0.5)
momentjs-rails (>= 2.9.0) momentjs-rails (>= 2.9.0)
mysql2 (~> 0.4.5)
octokit (~> 4.6, >= 4.6.2) octokit (~> 4.6, >= 4.6.2)
puma (~> 3.0) puma (~> 3.0)
rails (~> 5.0.0, >= 5.0.0.1) rails (~> 5.0.0, >= 5.0.0.1)
......
app/controllers/main_controller.rb
...@@ -32,7 +32,7 @@ class MainController < ApplicationController ...@@ -32,7 +32,7 @@ class MainController < ApplicationController
@selected_repos.each do |repo| @selected_repos.each do |repo|
row_table = Hash.new row_table = Hash.new
row_table[:name] = repo.name row_table[:name] = repo.name
row_table[:link] = repo.html_url row_table[:repo_id] = repo.id
row_table[:having_data] = false row_table[:having_data] = false
repo.issues.each do |issue| repo.issues.each do |issue|
if session[:filter_date].nil? || session[:filter_date] == "" || Date.parse(issue.created_at.to_s) >= Date.parse(session[:filter_date]) if session[:filter_date].nil? || session[:filter_date] == "" || Date.parse(issue.created_at.to_s) >= Date.parse(session[:filter_date])
......
app/controllers/repositories_controller.rb
...@@ -4,6 +4,26 @@ class RepositoriesController < ApplicationController ...@@ -4,6 +4,26 @@ class RepositoriesController < ApplicationController
@repos = Repository.all @repos = Repository.all
end end
def show
# byebug
@type = {
0 => "Open",
1 => "Closed",
2 => "Both"
}
@current_type = params["issue"]["type"]
@repo = Repository.find_by(id: params[:id])
@issues = nil
if @current_type.nil? || @current_type == "0"
@issues = @repo.issues.where("issues.closed_at IS NULL AND issues.is_pull = ?", false)
elsif @current_type == "1"
@issues = @repo.issues.where("issues.closed_at IS NOT NULL AND issues.is_pull = ?", false)
else
@issues = @repo.issues.where(is_pull: false)
end
end
def update def update
@repo = Repository.find_by(id: params[:id]) @repo = Repository.find_by(id: params[:id])
RepositoryJob.perform_now(@repo.name) RepositoryJob.perform_now(@repo.name)
......
app/controllers/settings_controller.rb
class SettingsController < ApplicationController class SettingsController < ApplicationController
def index def index
@repos = Repository.all @repos = Repository.all
@repos_to_update = Other.where(data_type: 2).map{ |x| x.data }
end end
def update_setting_1 def update_setting_1
$repos_to_update = params[:selected] @repos_to_update = params[:selected]
if !@repos_to_update.nil? && @repos_to_update.class == Array
Other.where(data_type: 2).delete_all
@repos_to_update.each do |repo|
Other.create(data: repo, data_type: 2)
end
end
end end
end end
app/helpers/application_helper.rb
module ApplicationHelper module ApplicationHelper
def format_duration_time(int_time)
mm, ss = int_time.divmod(60)
hh, mm = mm.divmod(60)
dd, hh = hh.divmod(24)
if dd != 0
dd.to_s + " days " + hh.to_s + " hours"
elsif hh != 0
hh.to_s + " hours " + mm.to_s + " minutes"
else
mm.to_s + " minutes " + ss.to_s + " seconds"
end
end
end end
app/helpers/issue_helper.rb
module IssueHelper module IssueHelper
include ApplicationHelper
end end
app/views/issues/show.html.erb
...@@ -42,98 +42,96 @@ ...@@ -42,98 +42,96 @@
</div> </div>
<% end %> <% end %>
<% end %> <% end %>
<table class="table table-bordered">
<thead>
<tr>
<th class="col-md-2">Label</th>
<th class="col-md-3">Duration</th>
<th class="col-md-7">Time</th>
</tr>
</thead>
<tbody>
<% @label_data.each do |key, value| %>
<tr>
<td><%= key %></td>
<% if value.nil? %>
<td>---</td>
<td>---</td>
<% else %>
<td><%= format_duration_time(value["time"].to_i) %></td>
<td>
<select>
<% value["data"].each do |x| %>
<option><%= x.started_time.to_s + " - " + x.end_time.to_s %></option>
<% end %>
</select>
</td>
<% end %>
</tr>
<% end %>
</tbody>
</table>
</div> </div>
<div class="col-md-4"> <div class="col-md-4">
<div> <div class="panel panel-default">
<div class="panel panel-default"> <div class="panel-heading">
<div class="panel-heading"> <h4>Assignee</h4>
<h4>Assignee</h4>
</div>
<ul class="list-group">
<% @issue.users.each do |user| %>
<li class="list-group-item"><%= user.login %></li>
<% end %>
</ul>
</div> </div>
<div class="panel panel-default"> <ul class="list-group">
<div class="panel-heading"> <% @issue.users.each do |user| %>
<h4>Label</h4> <li class="list-group-item"><%= user.login %></li>
</div> <% end %>
<ul class="list-group"> </ul>
<% @issue.labels.each do |label| %> </div>
<li class="list-group-item"><%= label.name %></li> <div class="panel panel-default">
<% end %> <div class="panel-heading">
</ul> <h4>Label</h4>
</div> </div>
<table class="table table-bordered"> <ul class="list-group">
<tbody> <% @issue.labels.each do |label| %>
<tr> <li class="list-group-item"><%= label.name %></li>
<td class="col-md-4">Started Time</td> <% end %>
<td class="col-md-8"> </ul>
<%= @started_time %> </div>
</td> <table class="table table-bordered">
</tr> <tbody>
<tr> <tr>
<td class="col-md-4">End Time</td> <td class="col-md-4">Started Time</td>
<td class="col-md-8"> <td class="col-md-8">
<%= @end_time %> <%= @started_time %>
</td> </td>
</tr> </tr>
<tr> <tr>
<td class="col-md-4">Duration</td> <td class="col-md-4">End Time</td>
<td class="col-md-8"> <td class="col-md-8">
<%= @duration %> <%= @end_time %>
</td> </td>
</tr> </tr>
<tr> <tr>
<td class="col-md-4">Estimate</td> <td class="col-md-4">Duration</td>
<td class="col-md-8">15 days</td> <td class="col-md-8">
</tr> <%= format_duration_time(@duration.to_i) %>
</tbody> </td>
</table> </tr>
<table class="table table-bordered"> <tr>
<thead> <td class="col-md-4">Estimate</td>
<tr> <td class="col-md-8">15 days</td>
<th class="col-md-4">Label</th> </tr>
<th class="col-md-4">Duration</th> </tbody>
<th class="col-md-4">Time</th> </table>
</tr> <div class="panel panel-default">
</thead> <div class="panel-heading">
<tbody> <h4>Pull Requests</h4>
<% @label_data.each do |key, value| %>
<tr>
<td><%= key %></td>
<% if value.nil? %>
<td>---</td>
<td>---</td>
<% else %>
<td><%= value["time"].to_i %></td>
<td>
<select>
<% value["data"].each do |x| %>
<option><%= x.started_time.to_s + " - " + x.end_time.to_s %></option>
<% end %>
</select>
</td>
<% end %>
</tr>
<% end %>
</tbody>
</table>
<div>
<div class="panel panel-default">
<div class="panel-heading">
<h4>Pull Requests</h4>
</div>
<ul class="list-group">
<% unless @pulls.nil? %>
<% @pulls.each do |pull| %>
<li class="list-group-item"><%= link_to pull.title, issue_path(pull) %></li>
<% end %>
<% end %>
</ul>
</div> </div>
<ul class="list-group">
<% unless @pulls.nil? %>
<% @pulls.each do |pull| %>
<li class="list-group-item"><%= link_to pull.title, issue_path(pull) %></li>
<% end %>
<% end %>
</ul>
</div> </div>
</div> </div>
</div> </div>
......
app/views/main/index.html.erb
...@@ -62,7 +62,7 @@ ...@@ -62,7 +62,7 @@
<tbody> <tbody>
<% @data_table.each do |row| %> <% @data_table.each do |row| %>
<tr> <tr>
<td class="col-md-2"><%= link_to row[:name], row[:link] %></td> <td class="col-md-2"><%= link_to row[:name], repository_path(row[:repo_id]) %></td>
<% @col_label_names.each do |label| %> <% @col_label_names.each do |label| %>
<% if row[label].nil? %> <% if row[label].nil? %>
<td class="col-md-1">0</td> <td class="col-md-1">0</td>
......
app/views/repositories/index.html.erb
...@@ -28,7 +28,7 @@ ...@@ -28,7 +28,7 @@
<tbody> <tbody>
<% @repos.each do |repo| %> <% @repos.each do |repo| %>
<tr> <tr>
<td><%= repo.name %></td> <td><%= link_to repo.name, repository_path(repo.id) %></td>
<td><%= repo.description %></td> <td><%= repo.description %></td>
<td> <td>
<%= form_for :repo, url: { action: "update", id: repo.id }, method: :patch, html: { class: "form-inline" } do |f| %> <%= form_for :repo, url: { action: "update", id: repo.id }, method: :patch, html: { class: "form-inline" } do |f| %>
......
app/views/repositories/show.html.erb 0 → 100644
<div class="container">
<h2><%= @repo.name %></h2>
<div id="toolbar">
<form class="form-inline">
<div class="form-group">
<label>Type</label>
<div class="form-group">
<%= select("issue", "type", @type.collect{ |x| [x[1], x[0]] }, {}, { class: "form-control" }) %>
<input type="submit" class="btn btn-default">
</div>
</div>
</form>
</div>
<div style="margin-top: 20px">
<table data-toolbar="#toolbar"
data-toggle="table"
data-search="true"
data-show-export="true"
data-show-columns="true"
data-show-export="true"
data-minimum-count-columns="2"
data-show-pagination-switch="true"
data-pagination="true"
data-id-field="id"
data-page-list="[10, 25, 50, 100, ALL]">
<thead>
<tr>
<th>List of issues</th>
</tr>
</thead>
<tbody>
<% @issues.each do |issue| %>
<tr>
<td><%= link_to issue.title, issue %></td>
</tr>
<% end %>
</tbody>
</table>
</div>
</div>
app/views/settings/index.html.erb
...@@ -12,8 +12,8 @@ ...@@ -12,8 +12,8 @@
</select> </select>
<button class="btn btn-default" id="btn1">Save</button> <button class="btn btn-default" id="btn1">Save</button>
<br><br> <br><br>
<% if $repos_to_update.class == Array %> <% if @repos_to_update.class == Array %>
<% $repos_to_update.each do |repo| %> <% @repos_to_update.each do |repo| %>
<span class="label label-primary"><%= repo %></span> <span class="label label-primary"><%= repo %></span>
<% end %> <% end %>
<% end %> <% end %>
...@@ -24,8 +24,8 @@ ...@@ -24,8 +24,8 @@
<script type="text/javascript"> <script type="text/javascript">
$(document).ready(function() { $(document).ready(function() {
$('#selectpicker').selectpicker(); // init picker $('#selectpicker').selectpicker(); // init picker
<% if $repos_to_update.class == Array %> <% if @repos_to_update.class == Array %>
$('#selectpicker').selectpicker('val', <%= $repos_to_update.to_s.html_safe %>); $('#selectpicker').selectpicker('val', <%= @repos_to_update.to_s.html_safe %>);
<% end %> <% end %>
}); });
......
config/database.yml
...@@ -5,21 +5,25 @@ ...@@ -5,21 +5,25 @@
# gem 'sqlite3' # gem 'sqlite3'
# #
default: &default default: &default
adapter: sqlite3 adapter: mysql2
encoding: utf8mb4
collation: utf8mb4_bin
database: ghr
username: root
password: 12345678
host: 127.0.0.1
port: 3306
pool: 5 pool: 5
timeout: 5000 timeout: 5000
development: development:
<<: *default <<: *default
database: db/development.sqlite3
# Warning: The database defined as "test" will be erased and # Warning: The database defined as "test" will be erased and
# re-generated from your development database when you run "rake". # re-generated from your development database when you run "rake".
# Do not set this db to the same as development or production. # Do not set this db to the same as development or production.
test: test:
<<: *default <<: *default
database: db/test.sqlite3
production: production:
<<: *default <<: *default
database: db/production.sqlite3
config/initializers/github.rb
$client = Octokit::Client.new(access_token: Rails.application.secrets.GITHUB_TOKEN) $client = Octokit::Client.new(access_token: Rails.application.secrets.GITHUB_TOKEN)
$repos_to_update = [] # $repos_to_update = []
db/migrate/20161125072356_create_issues.rb
...@@ -7,7 +7,7 @@ class CreateIssues < ActiveRecord::Migration[5.0] ...@@ -7,7 +7,7 @@ class CreateIssues < ActiveRecord::Migration[5.0]
t.string :title t.string :title
t.text :body t.text :body
t.references :repository, foreign_key: true t.references :repository, foreign_key: true
t.references :user, foreign_key: true t.references :user, foreign_key: true, null: true
t.date :created_at t.date :created_at
t.date :updated_at t.date :updated_at
......
db/migrate/20170105071411_change_collumn_body_in_review_comment.rb 0 → 100644
class ChangeCollumnBodyInReviewComment < ActiveRecord::Migration[5.0]
def change
change_column :review_comments, :body, :text
end
end
db/migrate/20170105072651_change_collumn_diff_hub_in_review_comment.rb 0 → 100644
class ChangeCollumnDiffHubInReviewComment < ActiveRecord::Migration[5.0]
def change
change_column :review_comments, :diff_hunk, :text
end
end
db/schema.rb
...@@ -10,55 +10,55 @@ ...@@ -10,55 +10,55 @@
# #
# It's strongly recommended that you check this file into your version control system. # It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema.define(version: 20161230092006) do ActiveRecord::Schema.define(version: 20170105072651) do
create_table "comments", force: :cascade do |t| create_table "comments", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin" do |t|
t.string "html_url" t.string "html_url"
t.text "body" t.text "body", limit: 65535
t.integer "issue_id" t.integer "issue_id"
t.integer "user_id" t.integer "user_id"
t.datetime "created_at", null: false t.datetime "created_at", null: false
t.datetime "updated_at", null: false t.datetime "updated_at", null: false
t.integer "score" t.integer "score"
t.index ["issue_id"], name: "index_comments_on_issue_id" t.index ["issue_id"], name: "index_comments_on_issue_id", using: :btree
t.index ["user_id"], name: "index_comments_on_user_id" t.index ["user_id"], name: "index_comments_on_user_id", using: :btree
end end
create_table "issues", force: :cascade do |t| create_table "issues", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin" do |t|
t.string "url" t.string "url"
t.string "repository_url" t.string "repository_url"
t.integer "number" t.integer "number"
t.string "title" t.string "title"
t.text "body" t.text "body", limit: 65535
t.integer "repository_id" t.integer "repository_id"
t.integer "user_id" t.integer "user_id"
t.datetime "created_at", null: false t.datetime "created_at", null: false
t.datetime "updated_at", null: false t.datetime "updated_at", null: false
t.string "html_url" t.string "html_url"
t.boolean "open" t.boolean "open"
t.date "closed_at" t.date "closed_at"
t.boolean "is_pull" t.boolean "is_pull"
t.integer "issue_id" t.integer "issue_id"
t.index ["issue_id"], name: "index_issues_on_issue_id" t.index ["issue_id"], name: "index_issues_on_issue_id", using: :btree
t.index ["repository_id"], name: "index_issues_on_repository_id" t.index ["repository_id"], name: "index_issues_on_repository_id", using: :btree
t.index ["user_id"], name: "index_issues_on_user_id" t.index ["user_id"], name: "index_issues_on_user_id", using: :btree
end end
create_table "issues_labels", id: false, force: :cascade do |t| create_table "issues_labels", id: false, force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin" do |t|
t.integer "issue_id", null: false t.integer "issue_id", null: false
t.integer "label_id", null: false t.integer "label_id", null: false
t.index ["issue_id", "label_id"], name: "index_issues_labels_on_issue_id_and_label_id" t.index ["issue_id", "label_id"], name: "index_issues_labels_on_issue_id_and_label_id", using: :btree
t.index ["label_id", "issue_id"], name: "index_issues_labels_on_label_id_and_issue_id" t.index ["label_id", "issue_id"], name: "index_issues_labels_on_label_id_and_issue_id", using: :btree
end end
create_table "issues_users", id: false, force: :cascade do |t| create_table "issues_users", id: false, force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin" do |t|
t.integer "issue_id", null: false t.integer "issue_id", null: false
t.integer "user_id", null: false t.integer "user_id", null: false
t.index ["issue_id", "user_id"], name: "index_issues_users_on_issue_id_and_user_id" t.index ["issue_id", "user_id"], name: "index_issues_users_on_issue_id_and_user_id", using: :btree
t.index ["user_id", "issue_id"], name: "index_issues_users_on_user_id_and_issue_id" t.index ["user_id", "issue_id"], name: "index_issues_users_on_user_id_and_issue_id", using: :btree
end end
create_table "labels", force: :cascade do |t| create_table "labels", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin" do |t|
t.string "url" t.string "url"
t.string "name" t.string "name"
t.string "color" t.string "color"
...@@ -66,10 +66,10 @@ ActiveRecord::Schema.define(version: 20161230092006) do ...@@ -66,10 +66,10 @@ ActiveRecord::Schema.define(version: 20161230092006) do
t.integer "repository_id" t.integer "repository_id"
t.datetime "created_at", null: false t.datetime "created_at", null: false
t.datetime "updated_at", null: false t.datetime "updated_at", null: false
t.index ["repository_id"], name: "index_labels_on_repository_id" t.index ["repository_id"], name: "index_labels_on_repository_id", using: :btree
end end
create_table "others", force: :cascade do |t| create_table "others", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin" do |t|
t.string "data" t.string "data"
t.string "data1" t.string "data1"
t.string "data2" t.string "data2"
...@@ -78,7 +78,7 @@ ActiveRecord::Schema.define(version: 20161230092006) do ...@@ -78,7 +78,7 @@ ActiveRecord::Schema.define(version: 20161230092006) do
t.datetime "updated_at", null: false t.datetime "updated_at", null: false
end end
create_table "repositories", force: :cascade do |t| create_table "repositories", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin" do |t|
t.string "name" t.string "name"
t.string "full_name" t.string "full_name"
t.string "description" t.string "description"
...@@ -90,25 +90,25 @@ ActiveRecord::Schema.define(version: 20161230092006) do ...@@ -90,25 +90,25 @@ ActiveRecord::Schema.define(version: 20161230092006) do
t.datetime "updated_at", null: false t.datetime "updated_at", null: false
end end
create_table "review_comments", force: :cascade do |t| create_table "review_comments", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin" do |t|
t.string "url" t.string "url"
t.string "diff_hunk" t.text "diff_hunk", limit: 65535
t.string "path" t.string "path"
t.string "body" t.text "body", limit: 65535
t.string "html_url" t.string "html_url"
t.string "pull_request_url" t.string "pull_request_url"
t.integer "issue_id" t.integer "issue_id"
t.integer "user_id" t.integer "user_id"
t.datetime "created_at", null: false t.datetime "created_at", null: false
t.datetime "updated_at", null: false t.datetime "updated_at", null: false
t.integer "rv_comment_id" t.integer "rv_comment_id"
t.string "commit_id" t.string "commit_id"
t.integer "score" t.integer "score"
t.index ["issue_id"], name: "index_review_comments_on_issue_id" t.index ["issue_id"], name: "index_review_comments_on_issue_id", using: :btree
t.index ["user_id"], name: "index_review_comments_on_user_id" t.index ["user_id"], name: "index_review_comments_on_user_id", using: :btree
end end
create_table "timelines", force: :cascade do |t| create_table "timelines", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin" do |t|
t.integer "duration" t.integer "duration"
t.integer "issue_id" t.integer "issue_id"
t.string "label" t.string "label"
...@@ -116,10 +116,10 @@ ActiveRecord::Schema.define(version: 20161230092006) do ...@@ -116,10 +116,10 @@ ActiveRecord::Schema.define(version: 20161230092006) do
t.datetime "end_time" t.datetime "end_time"
t.datetime "created_at", null: false t.datetime "created_at", null: false
t.datetime "updated_at", null: false t.datetime "updated_at", null: false
t.index ["issue_id"], name: "index_timelines_on_issue_id" t.index ["issue_id"], name: "index_timelines_on_issue_id", using: :btree
end end
create_table "users", force: :cascade do |t| create_table "users", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin" do |t|
t.string "login" t.string "login"
t.string "url" t.string "url"
t.string "html_url" t.string "html_url"
...@@ -127,4 +127,12 @@ ActiveRecord::Schema.define(version: 20161230092006) do ...@@ -127,4 +127,12 @@ ActiveRecord::Schema.define(version: 20161230092006) do
t.datetime "updated_at", null: false t.datetime "updated_at", null: false
end end
add_foreign_key "comments", "issues"
add_foreign_key "comments", "users"
add_foreign_key "issues", "repositories"
add_foreign_key "issues", "users"
add_foreign_key "labels", "repositories"
add_foreign_key "review_comments", "issues"
add_foreign_key "review_comments", "users"
add_foreign_key "timelines", "issues"
end end
lib/github_loader.rb
...@@ -11,7 +11,7 @@ class GithubLoader ...@@ -11,7 +11,7 @@ class GithubLoader
last_page = client.last_response.headers[:link].split(' ')[2][-3].to_i last_page = client.last_response.headers[:link].split(' ')[2][-3].to_i
end end
list_db_repo = [] # list_db_repo = []
for i in (1..last_page) for i in (1..last_page)
repos = client.org_repos($org, {type: 'all', page: i}) repos = client.org_repos($org, {type: 'all', page: i})
repos.each do |repo| repos.each do |repo|
...@@ -29,15 +29,17 @@ class GithubLoader ...@@ -29,15 +29,17 @@ class GithubLoader
db_repo.updated_at = repo.updated_at db_repo.updated_at = repo.updated_at
end end
list_db_repo.append(db_repo) db_repo.save
# list_db_repo.append(db_repo)
end end
end end
list_db_repo # list_db_repo
end end
def insert_labels(client, db_repos) def insert_labels(client)
$logger.info "Insert labels" $logger.info "Insert labels"
list_db_label = [] # list_db_label = []
db_repos = Repository.all
db_repos.each do |repo| db_repos.each do |repo|
labels = client.labels($org + "/" + repo.name) labels = client.labels($org + "/" + repo.name)
last_page = 1 last_page = 1
...@@ -48,22 +50,25 @@ class GithubLoader ...@@ -48,22 +50,25 @@ class GithubLoader
for i in (1..last_page) for i in (1..last_page)
labels = client.labels($org + "/" + repo.name, {page: i}) labels = client.labels($org + "/" + repo.name, {page: i})
labels.each do |label| labels.each do |label|
db_label = Label.find_by(id: label.id) normalized_label = label.name.gsub(/[^0-9A-Za-z]/, "").downcase # normalize string
db_label = Label.find_by(name: normalized_label)
if db_label.nil? if db_label.nil?
db_label = Label.new db_label = Label.new
db_label.id = label.id db_label.id = label.id
db_label.url = label.url db_label.url = label.url
db_label.repository_id = repo.id db_label.repository_id = repo.id
end end
db_label.name = label.name.gsub(/[^0-9A-Za-z]/, "").downcase # normalize string db_label.name = normalized_label
db_label.color = label.color db_label.color = label.color
db_label.default = label.default db_label.default = label.default
list_db_label.append(db_label) db_label.save
# list_db_label.append(db_label)
end end
end end
end end
list_db_label # list_db_label
end end
def insert_users(client) def insert_users(client)
...@@ -74,7 +79,7 @@ class GithubLoader ...@@ -74,7 +79,7 @@ class GithubLoader
last_page = client.last_response.headers[:link].split(' ')[2][-3].to_i last_page = client.last_response.headers[:link].split(' ')[2][-3].to_i
end end
list_db_user = [] # list_db_user = []
for i in (1..last_page) for i in (1..last_page)
users = client.org_members($org, {page: i}) users = client.org_members($org, {page: i})
users.each do |user| users.each do |user|
...@@ -87,15 +92,19 @@ class GithubLoader ...@@ -87,15 +92,19 @@ class GithubLoader
db_user.html_url = user.html_url db_user.html_url = user.html_url
end end
list_db_user.append(db_user) db_user.save
# list_db_user.append(db_user)
end end
end end
list_db_user # list_db_user
end end
def insert_issues(client, list_db_user, list_db_label, db_repos) def insert_issues(client)
$logger.info "Insert issues" $logger.info "Insert issues"
list_db_issue = [] list_db_user = User.all
list_db_label = Label.all
db_repos = Repository.all
# list_db_issue = []
db_repos.each do |repo| db_repos.each do |repo|
issues = client.list_issues($org + "/" + repo.name, { state: "all" }) issues = client.list_issues($org + "/" + repo.name, { state: "all" })
...@@ -116,6 +125,9 @@ class GithubLoader ...@@ -116,6 +125,9 @@ class GithubLoader
db_issue.number = issue.number db_issue.number = issue.number
db_issue.repository_id = repo.id db_issue.repository_id = repo.id
db_issue.user_id = issue.user.id db_issue.user_id = issue.user.id
if User.find_by(id: issue.user.id).nil?
next
end
db_issue.created_at = issue.created_at db_issue.created_at = issue.created_at
db_issue.html_url = issue.html_url db_issue.html_url = issue.html_url
if issue[:pull_request].nil? if issue[:pull_request].nil?
...@@ -136,8 +148,9 @@ class GithubLoader ...@@ -136,8 +148,9 @@ class GithubLoader
old_label = db_issue.labels.find_by(name: col_labels) old_label = db_issue.labels.find_by(name: col_labels)
db_issue.labels.clear db_issue.labels.clear
issue.labels.each do |label| issue.labels.each do |label|
label_name = label.name.gsub(/[^0-9A-Za-z]/, "").downcase
list_db_label.each do |db_label| list_db_label.each do |db_label|
if db_label.id == label.id if db_label.name == label_name
db_issue.labels.append(db_label) db_issue.labels.append(db_label)
break break
end end
...@@ -150,8 +163,12 @@ class GithubLoader ...@@ -150,8 +163,12 @@ class GithubLoader
end end
elsif new_label.name != old_label.name elsif new_label.name != old_label.name
timeline = Timeline.where(issue_id: db_issue.id, label: old_label.name).first timeline = Timeline.where(issue_id: db_issue.id, label: old_label.name).first
timeline.end_time = Time.now begin
timeline.save timeline.end_time = Time.now
timeline.save
rescue
byebug
end
new_timeline(db_issue.id, new_label.name) new_timeline(db_issue.id, new_label.name)
elsif new_label.name == old_label.name elsif new_label.name == old_label.name
...@@ -172,16 +189,19 @@ class GithubLoader ...@@ -172,16 +189,19 @@ class GithubLoader
end end
end end
list_db_issue.append(db_issue) db_issue.save
# list_db_issue.append(db_issue)
end end
end end
end end
list_db_issue # list_db_issue
end end
def insert_comments(client, list_db_repo) def insert_comments(client)
$logger.info "Insert comments" $logger.info "Insert comments"
list_db_comment = [] # list_db_comment = []
list_db_repo = Repository.all
list_db_repo.each do |db_repo| list_db_repo.each do |db_repo|
comments = client.issues_comments($org + "/" + db_repo.name) comments = client.issues_comments($org + "/" + db_repo.name)
...@@ -207,17 +227,19 @@ class GithubLoader ...@@ -207,17 +227,19 @@ class GithubLoader
db_comment.user_id = comment.user.id db_comment.user_id = comment.user.id
end end
db_comment.body = comment.body db_comment.body = comment.body
list_db_comment.append(db_comment) db_comment.save
# list_db_comment.append(db_comment)
end end
end end
end end
$logger.info "Size list_db_comment: " + list_db_comment.size.to_s # $logger.info "Size list_db_comment: " + list_db_comment.size.to_s
list_db_comment # list_db_comment
end end
def insert_review_comments(client, list_db_repo) def insert_review_comments(client)
$logger.info "Insert review comments" $logger.info "Insert review comments"
list_db_rv_comment = [] # list_db_rv_comment = []
list_db_repo = Repository.all
list_db_repo.each do |db_repo| list_db_repo.each do |db_repo|
rv_comments = client.pull_requests_comments($org + "/" + db_repo.name) rv_comments = client.pull_requests_comments($org + "/" + db_repo.name)
last_page = 1 last_page = 1
...@@ -241,6 +263,9 @@ class GithubLoader ...@@ -241,6 +263,9 @@ class GithubLoader
db_rv_comment.created_at = rv_comment.created_at db_rv_comment.created_at = rv_comment.created_at
unless rv_comment.user.nil? unless rv_comment.user.nil?
db_rv_comment.user_id = rv_comment.user.id db_rv_comment.user_id = rv_comment.user.id
if User.find_by(id: rv_comment.user.id).nil?
next
end
end end
pull_request_url_split = rv_comment.pull_request_url.split("/") pull_request_url_split = rv_comment.pull_request_url.split("/")
...@@ -252,11 +277,12 @@ class GithubLoader ...@@ -252,11 +277,12 @@ class GithubLoader
db_rv_comment.updated_at = rv_comment.updated_at db_rv_comment.updated_at = rv_comment.updated_at
db_rv_comment.body = rv_comment.body db_rv_comment.body = rv_comment.body
list_db_rv_comment.append(db_rv_comment) db_rv_comment.save
# list_db_rv_comment.append(db_rv_comment)
end end
end end
end end
list_db_rv_comment # list_db_rv_comment
end end
def update_issue_pull_association def update_issue_pull_association
...@@ -277,12 +303,6 @@ class GithubLoader ...@@ -277,12 +303,6 @@ class GithubLoader
end end
end end
# def update_repo(repositories)
# repositories.each do |repo|
# end
# end
def new_timeline(issue_id, label) def new_timeline(issue_id, label)
timeline = Timeline.new timeline = Timeline.new
timeline.issue_id = issue_id timeline.issue_id = issue_id
......
lib/tasks/github.rake
...@@ -16,42 +16,49 @@ namespace :github do ...@@ -16,42 +16,49 @@ namespace :github do
$logger.info "Begin task insert database from github" $logger.info "Begin task insert database from github"
start_time = Time.now start_time = Time.now
list_db_repo = loader.insert_repos($client) loader.insert_repos($client)
list_db_user = loader.insert_users($client) loader.insert_users($client)
list_db_label = loader.insert_labels($client, list_db_repo) loader.insert_labels($client)
list_db_issue = loader.insert_issues($client, list_db_user, list_db_label, list_db_repo) loader.insert_issues($client)
list_db_comment = loader.insert_comments($client, list_db_repo)
list_db_rv_comment = loader.insert_review_comments($client, list_db_repo)
$logger.info "begin transaction" loader.update_issue_pull_association
start_time_transaction = Time.now
ActiveRecord::Base.transaction do
list_db_repo.each do |db_repo| # $logger.info "begin transaction 1"
db_repo.save # start_time_transaction = Time.now
end # ActiveRecord::Base.transaction do
list_db_user.each do |db_user| # list_db_repo.each do |db_repo|
db_user.save # db_repo.save
end # end
list_db_label.each do |db_label| # list_db_user.each do |db_user|
db_label.save # db_user.save
end # end
list_db_issue.each do |db_issue| # list_db_label.each do |db_label|
db_issue.save # db_label.save
end # end
list_db_comment.each do |db_comment| # list_db_issue.each do |db_issue|
db_comment.save # db_issue.save
end # end
# end
list_db_rv_comment.each do |db_rv_comment| # $logger.info "Finished transaction1 in #{Time.now - start_time_transaction}s"
db_rv_comment.save
end # list_db_comment = loader.insert_comments($client, list_db_repo)
end # list_db_rv_comment = loader.insert_review_comments($client, list_db_repo)
$logger.info "Finished transaction in #{Time.now - start_time_transaction}s"
# start_time_transaction = Time.now
# ActiveRecord::Base.transaction do
# list_db_comment.each do |db_comment|
# db_comment.save
# end
# list_db_rv_comment.each do |db_rv_comment|
# db_rv_comment.save
# end
# end
# $logger.info "Finished transaction2 in #{Time.now - start_time_transaction}s"
$logger.info "Finished task import in #{Time.now - start_time}s" $logger.info "Finished task import in #{Time.now - start_time}s"
end end
...@@ -59,15 +66,16 @@ namespace :github do ...@@ -59,15 +66,16 @@ namespace :github do
task insert_github_data_for_specific_repos: :environment do task insert_github_data_for_specific_repos: :environment do
loader = GithubLoader.new loader = GithubLoader.new
start_time = Time.now start_time = Time.now
$logger.info "Begin task update " + $repos_to_update.to_s repos_to_update = Other.where(data_type: 2).map{ |x|x.data }
list_db_repo = Repository.where(name: $repos_to_update) $logger.info "Begin task update " + repos_to_update.to_s
list_db_repo = Repository.where(name: repos_to_update)
list_db_user = User.all list_db_user = User.all
list_db_label = Label.all list_db_label = Label.all
list_db_issue = loader.insert_issues($client, list_db_user, list_db_label, list_db_repo) list_db_issue = loader.insert_issues($client, list_db_user, list_db_label, list_db_repo)
list_db_comment = loader.insert_comments($client, list_db_repo) list_db_comment = loader.insert_comments($client, list_db_repo)
list_db_rv_comment = loader.insert_review_comments($client, list_db_repo) list_db_rv_comment = loader.insert_review_comments($client, list_db_repo)
logger.info "begin transaction" $logger.info "begin transaction"
start_time_transaction = Time.now start_time_transaction = Time.now
ActiveRecord::Base.transaction do ActiveRecord::Base.transaction do
...@@ -83,8 +91,8 @@ namespace :github do ...@@ -83,8 +91,8 @@ namespace :github do
db_rv_comment.save db_rv_comment.save
end end
end end
logger.info "Finished transaction in #{Time.now - start_time_transaction}s" $logger.info "Finished transaction in #{Time.now - start_time_transaction}s"
logger.info "Finished task update in #{Time.now - start_time}s" $logger.info "Finished task update in #{Time.now - start_time}s"
end end
desc "Insert Repository only" desc "Insert Repository only"
...@@ -100,10 +108,6 @@ namespace :github do ...@@ -100,10 +108,6 @@ namespace :github do
db_repo.save db_repo.save
end end
# list_db_user.each do |db_user|
# db_user.save
# end
list_db_label.each do |db_label| list_db_label.each do |db_label|
db_label.save db_label.save
end end
...@@ -116,21 +120,20 @@ namespace :github do ...@@ -116,21 +120,20 @@ namespace :github do
loader = GithubLoader.new loader = GithubLoader.new
$logger.info "Begin task insert comments" $logger.info "Begin task insert comments"
start_time = Time.now start_time = Time.now
list_db_repo = Repository.all loader.insert_comments($client)
list_db_comment = loader.insert_comments($client, list_db_repo) loader.insert_review_comments($client)
list_db_rv_comment = loader.insert_review_comments($client, list_db_repo)
# start_time_transaction = Time.now
start_time_transaction = Time.now # ActiveRecord::Base.transaction do
ActiveRecord::Base.transaction do # list_db_comment.each do |db_comment|
list_db_comment.each do |db_comment| # db_comment.save
db_comment.save # end
end
# list_db_rv_comment.each do |db_rv_comment|
list_db_rv_comment.each do |db_rv_comment| # db_rv_comment.save
db_rv_comment.save # end
end # end
end # $logger.info "Finished transaction in #{Time.now - start_time_transaction}s"
$logger.info "Finished transaction in #{Time.now - start_time_transaction}s"
$logger.info "Finished task import comments in #{Time.now - start_time}s" $logger.info "Finished task import comments in #{Time.now - start_time}s"
end end
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment