Merge branch 'feature/improve-performance-workflow-report' into 'hotfix/workflow-report-plugin'

Feature/improve performance workflow report

See merge request !12

plugins/workflow_report/app/controllers/workflow_report_controller.rb

@@ -48,6 +48,7 @@ class WorkflowReportController < ApplicationController
     result = Rails.cache.fetch("#{team}_#{params[:year]}_#{params[:month]}", expires_in: 1.hours) do
       WorkflowReport.build_report(params[:year].to_i, params[:month].to_i, project_ids)
     end
+
     respond_to do |format|
       format.js { render 'build_table', locals: { result: result[:workflow_report], error_links: result[:error_links], thead: TABLE_HEADER } }
     end

plugins/workflow_report/app/models/workflow_report_custom_value.rb

+class WorkflowReportCustomValue < CustomValue
+  belongs_to :workflow_report_issue, class_name: 'WorkflowReportCustomValue', foreign_key: 'custom_field_id'
+end

plugins/workflow_report/app/models/workflow_report_issue.rb

 class WorkflowReportIssue < Issue
-  TESTCASE_FIELD_ID = 22
-  BUGS_FIELD_ID = 23
-  STG_BUGS_FIELD_ID = 27
-  PROD_BUGS_FIELD_ID = 28
-  PR_FIELD_ID = 18
-  JP_REQUEST_FIELD_ID = 16
+  BUG_TRACKER_ID = 1
+  CLOSED_STATUS_PROJECT = 9
+  belongs_to :workflow_report_version, class_name: 'Version', foreign_key: 'fixed_version_id'
+  has_many :workflow_report_custom_values, class_name: 'WorkflowReportCustomValue', foreign_key: 'customized_id'
 
-  scope :find_root_ids, ->(year, month, project_ids) {
-    joins(:project, project: :enabled_modules)
-      .joins('LEFT OUTER JOIN time_entries ON issues.id = time_entries.issue_id')
-      .where.not(projects: { status: 9 })
+  scope :raw_tasks_records, -> (year, month, project_ids) {
+    includes(:project, :workflow_report_version, :workflow_report_custom_values, :time_entries, :status, project: :enabled_modules)
+      .where.not(projects: { status: CLOSED_STATUS_PROJECT }, issues: { tracker_id: BUG_TRACKER_ID })
       .where(projects: { id: project_ids }, enabled_modules: { name: 'time_tracking' })
       .where(['((time_entries.spent_on IS NOT NULL AND time_entries.tyear = ? AND time_entries.tmonth = ?) OR (issues.closed_on BETWEEN ? AND ?))',
               year, month, DateTime.new(year, month).beginning_of_day, DateTime.new(year, month, -1).end_of_day])
-      .distinct.pluck(:root_id)
-  }
-
-  scope :find_sum_hours_records, ->(root_ids) {
-    select(:root_id, 'sum(time_entries.hours) as hours', 'max(IFNULL(c1.value,-1)) as testcases', 'max(IFNULL(c2.value,-1)) as bugs', 'max(IFNULL(stg.value,-1)) as stg_bugs', 'max(IFNULL(prod.value,-1)) as prod_bugs')
-      .joins(:time_entries)
-      .joins("LEFT OUTER JOIN custom_values c1 ON c1.customized_id = issues.id and c1.custom_field_id=#{TESTCASE_FIELD_ID}")
-      .joins("LEFT OUTER JOIN custom_values c2 ON c2.customized_id = issues.id and c2.custom_field_id=#{BUGS_FIELD_ID}")
-      .joins("LEFT OUTER JOIN custom_values stg ON stg.customized_id = issues.id and stg.custom_field_id=#{STG_BUGS_FIELD_ID}")
-      .joins("LEFT OUTER JOIN custom_values prod ON prod.customized_id = issues.id and prod.custom_field_id=#{PROD_BUGS_FIELD_ID}")
-      .where(issues: { root_id: root_ids })
-      .where.not(issues: { tracker_id: 1 })
-      .group(:root_id)
-  }
-
-  scope :raw_tasks_records, ->(root_ids) {
-    select(:root_id, :id, :tracker_id, :subject, :due_date, :created_on, :closed_on, :estimated_hours, 'issue_statuses.name as status', 'versions.name as target_version', 'projects.name as project', 'pr.value as pr', 'jr.value as jp_request')
-      .joins(:project, :status)
-      .joins('LEFT JOIN versions ON issues.fixed_version_id = versions.id')
-      .joins("LEFT OUTER JOIN custom_values jr ON jr.customized_id = issues.id and jr.custom_field_id=#{JP_REQUEST_FIELD_ID}")
-      .joins("LEFT OUTER JOIN custom_values pr ON pr.customized_id = issues.id and pr.custom_field_id=#{PR_FIELD_ID}")
-      .where(issues: { root_id: root_ids })
-      .where.not(issues: { tracker_id: 1 })
       .order(:root_id)
   }
 end

plugins/workflow_report/app/models/workflow_report_journal.rb

 class WorkflowReportJournal < Journal
-  scope :find_journal_by_issue_ids, ->(issue_ids) {
-    joins(:details)
+  scope :find_journal_by_issue_ids, -> (issue_ids) {
+    includes(:details)
       .where(journals: { journalized_id: issue_ids }, journal_details: { prop_key: 'estimated_hours' })
       .order(:journalized_id, :id)
       .pluck(:journalized_id, :old_value, :value, :created_on, :notes)

plugins/workflow_report/app/models/workflow_report_version.rb

+class WorkflowReportVersion < Version
+  has_many :workflow_report_issues, foreign_key: 'fixed_version_id'
+end

plugins/workflow_report/lib/workflow_report.rb

@@ -5,46 +5,47 @@ module WorkflowReport
   EST_DETAIL_FIRST_COL = 14
   ACTUAL_TIME_DETAIL_FIRST_COL = 21
   PR_CMT_COL = 34
+  PR_FIELD_ID = 18
+  JP_REQUEST_FIELD_ID = 16
+  BUG_COLS = 28
+  USER_STORY_TRACKER_ID = 12
+  COL_REQUIREMENT_TO_RELEASE = 6
   PROCESS = ['1. Requirement', '2. Design', '3. Coding', '4. Testing', '5. Bug fixing', '6. Release', ''].freeze
-  BUGS = { testcases: 28, bugs: 29, stg_bugs: 30, prod_bugs: 31 }.freeze
+  BUGS = { testcases: 22, bugs: 23, stg_bugs: 27, prod_bugs: 28 }.freeze
 
   class << self
     def build_report(year, month, project_ids)
       github = Github.new oauth_token: $workflow_report_config['github_token']
       error_links = []
+      github_links = { prs: [], issues: [] }
       result = TABLE_HEADER.length.times.map { [] }
-      root_ids = WorkflowReportIssue.find_root_ids(year, month, project_ids)
-      return { workflow_report: [], error_links: [] } if root_ids.empty?
+      raw_tasks = WorkflowReportIssue.raw_tasks_records(year, month, project_ids)
+      return { workflow_report: [], error_links: [] } if raw_tasks.empty?
 
-      sum_hours_records = WorkflowReportIssue.find_sum_hours_records(root_ids)
-      raw_tasks = WorkflowReportIssue.raw_tasks_records(root_ids)
-
-      raw_tasks.group_by(&:root_id).each do |root_id, record|
-        sum_hours_record = sum_hours_records.find { |hr| hr.root_id == root_id }
-
-        issue_ids = record.map(&:id)
+      raw_tasks.group_by(&:root_id).each do |root_id, issues|
+        issue_ids = issues.map(&:id)
+        root_issue = WorkflowReportIssue.find(root_id)
         journals = WorkflowReportJournal.find_journal_by_issue_ids(issue_ids).to_a
 
         result[0] << root_id
-        result[1] << record.first[:project].gsub(/[^[:print:]]/, '')
-        result[2] << record.first[:subject].gsub(/[^[:print:]]/, '')
-        result[3] << record.first[:target_version]
-        issue_created_on = record.min_by { |i| i[:created_on] if i[:created_on].present? }
-        result[4].push(issue_created_on.present? ? issue_created_on[:created_on]&.strftime('%Y-%m-%d %H:%M:%S') : '')
-        closed_on_issues = record.map(&:closed_on).compact
+        result[1] << issues.first.project.name.gsub(/[^[:print:]]/, '')
+        result[2] << root_issue&.subject.gsub(/[^[:print:]]/, '')
+        result[3] << issues.first.workflow_report_version&.name
+        issue_created_on = issues.min_by { |i| i[:created_on] if i[:created_on].present? }
+        result[4].push(issue_created_on.present? ? issue_created_on.created_on&.strftime('%Y-%m-%d %H:%M:%S') : '')
+        closed_on_issues = issues.map(&:closed_on).compact
         issue_closed_on = closed_on_issues.max_by { |close_on| close_on }
         result[5].push(issue_closed_on.present? ? issue_closed_on.strftime('%Y-%m-%d %H:%M:%S') : '')
-
-        issue_due_dates = record.map(&:due_date).compact
+        issue_due_dates = issues.map(&:due_date).compact
         issue_due_date = issue_due_dates.max_by { |due_date| due_date }
-
         result[6].push(issue_due_date.present? ? issue_due_date&.strftime('%Y-%m-%d %H:%M:%S') : '')
-        result[7].push(record.first[:status].present? ? record.first[:status] : '')
-        sum_estimated_hours = record.map { |i| i[:estimated_hours] }.compact.sum
-        result[8] << sum_estimated_hours
-        if sum_hours_record.present?
-          result[9] << sum_hours_record[:hours]
-          result[10] << (sum_hours_record[:hours] - sum_estimated_hours)
+        result[7].push(root_issue.status&.name.present? ? root_issue.status&.name : '')
+        sum_estimated_hours = issues.map(&:estimated_hours).compact.sum
+        result[8] << sum_estimated_hours.round(2)
+        actual_time = issues.map { |issue| issue.time_entries.sum(:hours) }.sum
+        if actual_time.present?
+          result[9] << actual_time.round(2)
+          result[10] << (actual_time - sum_estimated_hours).round(2)
         else
           result[9] << ''
           result[10] << ''
@@ -60,12 +61,12 @@ module WorkflowReport
         else
           (11..13).each { |i| result[i] << '' }
         end
-        pull_request = ''
-        jp_request = ''
-        record.each do |issue|
-          if issue.tracker_id == 12
-            pull_request = issue.pr
-            jp_request = issue.jp_request
+        issues.each do |issue|
+          if issue.tracker_id == USER_STORY_TRACKER_ID
+            pr_links = issue.custom_values.find_by(custom_field_id: PR_FIELD_ID)&.value
+            jp_request = issue.custom_values.find_by(custom_field_id: JP_REQUEST_FIELD_ID)&.value
+            github_links[:prs].push({ row: result[0].length - 1, links: pr_links, root_id: root_id }) if pr_links.present?
+            github_links[:issues].push({ row: result[0].length - 1, links: jp_request, root_id: root_id }) if jp_request.present?
           end
           process = get_process(issue.subject.gsub(/[^[:print:]]/, ''))
 
@@ -80,71 +81,39 @@ module WorkflowReport
               end
             end
           else
-            6.times do |index|
+            COL_REQUIREMENT_TO_RELEASE.times do |index|
               build_est_to_actual_time!(result, EST_DETAIL_FIRST_COL, 0, index)
               build_est_to_actual_time!(result, ACTUAL_TIME_DETAIL_FIRST_COL, 0, index)
             end
-            build_est_to_actual_time!(result, EST_DETAIL_FIRST_COL, issue.estimated_hours.to_f, 6)
-            build_est_to_actual_time!(result, ACTUAL_TIME_DETAIL_FIRST_COL, issue.spent_hours.to_f, 6)
+            build_est_to_actual_time!(result, EST_DETAIL_FIRST_COL, issue.estimated_hours.to_f, COL_REQUIREMENT_TO_RELEASE)
+            build_est_to_actual_time!(result, ACTUAL_TIME_DETAIL_FIRST_COL, issue.spent_hours.to_f, COL_REQUIREMENT_TO_RELEASE)
           end
         end
-        if sum_hours_record.nil?
+        if sum_estimated_hours.zero?
           # testcases, internal bug, stg bug,  prod bug
           (28..31).each { |i| result[i] << '' }
         else
-          # testcases, internal bug, stg bug,  prod bug
-          BUGS.each do |status, column|
-            result[column].push(sum_hours_record[status].to_i >= 0 ? sum_hours_record[status] : '')
-          end
-        end
-        if jp_request.strip.blank?
-          result[32] << ''
-          result[33] << ''
-        else
-          jp_request = jp_request.strip
-          result[32] << jp_request
-          begin
-            jp_request_arr = URI(jp_request).path.split('/').reject(&:blank?)
-            if jp_request_arr.length >= 4 && jp_request_arr[3].to_i.positive?
-              issue_detail = github.issues.find user: jp_request_arr[0], repo: jp_request_arr[1], number: jp_request_arr[3]
-              result[33].push(issue_detail.success? ? issue_detail.comments : '')
-            else
-              result[33].push('') if result[33][result[0].length - 1].nil?
-            end
-          rescue StandardError => e
-            error_links << "##{root_id} Error occurred during the build with the ISSUE link:\r\n - #{jp_request} \r\n - #{e.message.gsub(/(?<=access_token=).*/, '123')}"
-            result[33].push('') if result[33][result[0].length - 1].nil?
+          BUGS.each_with_index do |(_key, custom_id), index|
+            result[BUG_COLS + index] << find_max_value_by_custom_field_id(issues, custom_id)
           end
         end
-        # pr detail
-        pr = { pr_comments: 0, pr_review_comments: 0, pr_commits: 0, pr_changed_files: 0, pr_additions: 0, pr_deletions: 0 }
-
-        if pull_request.present?
-          prs = pull_request.split("\r\n").compact
-          prs.each do |link|
-            begin
-              pr_link_arr = URI(link.strip).path.split('/').reject!(&:empty?)
-              next unless pr_link_arr.length >= 4
-
-              pr_detail = github.pull_requests.find user: pr_link_arr[0], repo: pr_link_arr[1], number: pr_link_arr[3]
-              if pr_detail.success?
-                pr[:pr_comments] += pr_detail.comments
-                pr[:pr_review_comments] += pr_detail.review_comments
-                pr[:pr_commits] += pr_detail.commits
-                pr[:pr_additions] += pr_detail.additions
-                pr[:pr_deletions] += pr_detail.deletions
-                pr[:pr_changed_files] += pr_detail.changed_files
-              end
-            rescue StandardError => e
-              error_links << "##{root_id} Error occurred during the build with the PR link:\r\n - #{link} \r\n - #{e.message.gsub(/(?<=access_token=).*/, '123')}"
-            end
-          end
+      end
+
+      threads_pr = github_links[:prs].map do |prs|
+        Thread.new do
+          find_detail_pr(github, prs[:links], error_links, result, prs[:row], prs[:root_id])
         end
-        pr.each_with_index do |(key, _detail), index|
-          result[PR_CMT_COL + index] << pr[key]
+      end
+
+      threads_issue = github_links[:issues].map do |issues|
+        Thread.new do
+          find_detail_issue(github, result, error_links, issues[:links], issues[:row], issues[:root_id])
         end
       end
 
+      threads_issue.each(&:join)
+      threads_pr.each(&:join)
+
       { workflow_report: result, error_links: error_links }
     end
 
@@ -166,17 +135,17 @@ module WorkflowReport
           end
         else
           if name.include? "Requirement"
-            name = "1. " + name
+            name = "1. #{name}"
           elsif name.include? "Design"
-            name = "2. " + name
+            name = "2. #{name}"
           elsif ["Coding", "Code review"].include? name
             name = "3. Coding"
           elsif ["Testing", "Create Test case"].include? name
             name = "4. Testing"
           elsif name.include? "Bug fixing"
-            name = "5. " + name
+            name = "5. #{name}"
           elsif name.include? "Release"
-            name = "6. " + name
+            name = "6. #{name}"
           end
         end
       end
@@ -186,9 +155,75 @@ module WorkflowReport
 
     def build_est_to_actual_time!(result, column, time, index)
       if result[column + index][result[0].length - 1].present?
-        result[column + index][result[0].length - 1] = result[column + index][result[0].length - 1] + time
+        result[column + index][result[0].length - 1] = result[column + index][result[0].length - 1] + time.round(2)
       else
-        result[column + index].push(time)
+        result[column + index].push(time.round(2))
+      end
+    end
+
+    def find_max_value_by_custom_field_id(issues, custom_field_id)
+      max_value = 0
+      issue_custom = []
+      issues.each do |issue|
+        filtered_values = issue.workflow_report_custom_values.select { |custom_value| custom_value.custom_field_id == custom_field_id }
+        next if filtered_values.empty?
+
+        issue_custom = filtered_values
+      end
+
+      max_value_in_record = issue_custom.map { |custom| custom.value.to_i }.max
+
+      [max_value, max_value_in_record.to_i].max
+    end
+
+    def find_detail_pr(github, pull_request, error_links, result, row, root_id)
+      pr = { pr_comments: 0, pr_review_comments: 0, pr_commits: 0, pr_changed_files: 0, pr_additions: 0, pr_deletions: 0 }
+
+      if pull_request.present?
+        prs = pull_request.split("\r\n").compact
+        prs.each do |link|
+          begin
+            pr_link_arr = URI(link.strip).path.split('/').reject!(&:empty?)
+            next unless pr_link_arr.length >= 4
+
+            pr_detail = github.pull_requests.find user: pr_link_arr[0], repo: pr_link_arr[1], number: pr_link_arr[3]
+            if pr_detail.success?
+              pr[:pr_comments] += pr_detail.comments
+              pr[:pr_review_comments] += pr_detail.review_comments
+              pr[:pr_commits] += pr_detail.commits
+              pr[:pr_additions] += pr_detail.additions
+              pr[:pr_deletions] += pr_detail.deletions
+              pr[:pr_changed_files] += pr_detail.changed_files
+            end
+          rescue StandardError => e
+            error_links << "##{root_id} Error occurred during the build with the PR link:\r\n - #{link} \r\n - #{e.message.gsub(/(?<=access_token=).*/, '123')}"
+          end
+        end
+      end
+      pr.each_with_index do |(key, _detail), index|
+        result[PR_CMT_COL + index][row] = pr[key]
+      end
+    end
+
+    def find_detail_issue(github, result, error_links, jp_request, row, root_id)
+      if jp_request.strip.blank?
+        result[32][row] = ''
+        result[33][row] = ''
+      else
+        jp_request = jp_request.strip
+        result[32][row] = jp_request
+        begin
+          jp_request_arr = URI(jp_request).path.split('/').reject(&:blank?)
+          if jp_request_arr.length >= 4 && jp_request_arr[3].to_i.positive?
+            issue_detail = github.issues.find user: jp_request_arr[0], repo: jp_request_arr[1], number: jp_request_arr[3]
+            result[33][row] = issue_detail.success? ? issue_detail.comments : ''
+          else
+            result[33][row] = '' if result[33][row].nil?
+          end
+        rescue StandardError => e
+          error_links << "##{root_id} Error occurred during the build with the ISSUE link:\r\n - #{jp_request} \r\n - #{e.message.gsub(/(?<=access_token=).*/, '123')}"
+          result[33][row] = '' if result[33][row].nil?
+        end
       end
     end
   end