diff --git a/app/avo/resources/commit.rb b/app/avo/resources/commit.rb index 5ee00d0..ef8aded 100644 --- a/app/avo/resources/commit.rb +++ b/app/avo/resources/commit.rb @@ -4,7 +4,7 @@ class Avo::Resources::Commit < Avo::BaseResource # self.search = { # query: -> { query.ransack(id_eq: params[:q], m: "or").result(distinct: false) } # } - + def fields field :id, as: :id field :sha, as: :text diff --git a/app/avo/resources/repo_host_event.rb b/app/avo/resources/repo_host_event.rb index 80c6006..484f9fa 100644 --- a/app/avo/resources/repo_host_event.rb +++ b/app/avo/resources/repo_host_event.rb @@ -4,7 +4,7 @@ class Avo::Resources::RepoHostEvent < Avo::BaseResource # self.search = { # query: -> { query.ransack(id_eq: params[:q], m: "or").result(distinct: false) } # } - + def fields field :id, as: :id field :user, as: :belongs_to diff --git a/app/controllers/admin/base_controller.rb b/app/controllers/admin/base_controller.rb index e2aba0d..74fa9e3 100644 --- a/app/controllers/admin/base_controller.rb +++ b/app/controllers/admin/base_controller.rb @@ -8,4 +8,4 @@ class Admin::BaseController < ApplicationController redirect_to root_path, alert: "You are not authorized to access this page." end end -end \ No newline at end of file +end diff --git a/app/controllers/admin/timeline_controller.rb b/app/controllers/admin/timeline_controller.rb index 0d51ae7..e213bc5 100644 --- a/app/controllers/admin/timeline_controller.rb +++ b/app/controllers/admin/timeline_controller.rb @@ -13,18 +13,18 @@ class Admin::TimelineController < Admin::BaseController @prev_date = @date - 1.day # User selection logic - raw_user_ids = params[:user_ids].present? ? params[:user_ids].split(',').map(&:to_i).uniq : [] - + raw_user_ids = params[:user_ids].present? ? params[:user_ids].split(",").map(&:to_i).uniq : [] + # Always include current_user (admin) - @selected_user_ids = [current_user.id] + raw_user_ids + @selected_user_ids = [ current_user.id ] + raw_user_ids @selected_user_ids.uniq! - + user_ids_to_fetch = @selected_user_ids # Fetch all valid users in one go users_by_id = User.where(id: user_ids_to_fetch).index_by(&:id) # Ensure we only use IDs of users that actually exist - valid_user_ids_to_fetch = users_by_id.keys + valid_user_ids_to_fetch = users_by_id.keys mappings_by_user_project = ProjectRepoMapping.where(user_id: valid_user_ids_to_fetch) .group_by(&:user_id) @@ -37,7 +37,7 @@ class Admin::TimelineController < Admin::BaseController all_heartbeats = Heartbeat .where(user_id: valid_user_ids_to_fetch, deleted_at: nil) - .where('time >= ? AND time <= ?', start_of_day_timestamp, end_of_day_timestamp) + .where("time >= ? AND time <= ?", start_of_day_timestamp, end_of_day_timestamp) .select(:id, :user_id, :time, :entity, :project, :editor, :language) .order(:user_id, :time) .to_a @@ -48,7 +48,7 @@ class Admin::TimelineController < Admin::BaseController users_to_process.each do |user| user_daily_heartbeats_relation = Heartbeat.where(user_id: user.id, deleted_at: nil) - .where('time >= ? AND time <= ?', start_of_day_timestamp, end_of_day_timestamp) + .where("time >= ? AND time <= ?", start_of_day_timestamp, end_of_day_timestamp) total_coded_time_seconds = user_daily_heartbeats_relation.duration_seconds user_heartbeats_for_spans = heartbeats_by_user_id[user.id] || [] @@ -68,7 +68,7 @@ class Admin::TimelineController < Admin::BaseController span_duration = last_hb_time_numeric - start_time_numeric # This is span length, not necessarily active coding time within span span_duration = 0 if span_duration < 0 - files = current_span_heartbeats.map { |h| h.entity&.split('/')&.last }.compact.uniq + files = current_span_heartbeats.map { |h| h.entity&.split("/")&.last }.compact.uniq projects_edited_details_for_span = [] unique_project_names_in_current_span = current_span_heartbeats.map(&:project).compact.reject(&:blank?).uniq @@ -105,7 +105,7 @@ class Admin::TimelineController < Admin::BaseController total_coded_time: total_coded_time_seconds # Actual coded time for the user for the day } end - + # Order @users_with_timeline_data according to @selected_user_ids # This ensures that if a user was explicitly selected they appear in the timeline # even if they have no heartbeats for the day. @@ -145,15 +145,15 @@ class Admin::TimelineController < Admin::BaseController def leaderboard_users period = params[:period] limit = 25 - - leaderboard_period_type = (period == 'last_7_days') ? :last_7_days : :daily + + leaderboard_period_type = (period == "last_7_days") ? :last_7_days : :daily start_date = Date.current # Leaderboard job for :last_7_days uses Date.current as start_date leaderboard = Leaderboard.where.not(finished_generating_at: nil) .find_by(start_date: start_date, period_type: leaderboard_period_type, deleted_at: nil) user_ids_from_leaderboard = leaderboard ? leaderboard.entries.order(total_seconds: :desc).limit(limit).pluck(:user_id) : [] - + all_ids_to_fetch = user_ids_from_leaderboard.dup all_ids_to_fetch.unshift(current_user.id).uniq! @@ -169,14 +169,14 @@ class Admin::TimelineController < Admin::BaseController # Add leaderboard users, ensuring no duplicates and respecting limit user_ids_from_leaderboard.each do |uid| - break if final_user_objects.size >= limit - next if uid == current_user.id - + break if final_user_objects.size >= limit + next if uid == current_user.id + if user_data = users_data[uid] final_user_objects << { id: user_data.id, display_name: user_data.display_name, avatar_url: user_data.avatar_url } end end - + render json: { users: final_user_objects } end -end \ No newline at end of file +end diff --git a/app/jobs/process_commit_job.rb b/app/jobs/process_commit_job.rb index 52d1de0..ea0bf02 100644 --- a/app/jobs/process_commit_job.rb +++ b/app/jobs/process_commit_job.rb @@ -1,5 +1,5 @@ -require 'http' -require 'json' +require "http" +require "json" class ProcessCommitJob < ApplicationJob queue_as :literally_whenever @@ -26,7 +26,7 @@ class ProcessCommitJob < ApplicationJob # and the commit record already exists (e.g., adding gitlab_raw to an existing commit) return end - + Rails.logger.info "[ProcessCommitJob] Processing commit #{commit_sha} for User ##{user_id} via #{provider_sym} from URL: #{commit_api_url}" case provider_sym @@ -57,19 +57,19 @@ class ProcessCommitJob < ApplicationJob if response.status.success? commit_data_json = response.parse - - api_commit_sha = commit_data_json['sha'] + + api_commit_sha = commit_data_json["sha"] unless api_commit_sha == commit_sha Rails.logger.error "[ProcessCommitJob] SHA mismatch for User ##{user.id}. Expected #{commit_sha}, API returned #{api_commit_sha}. URL: #{commit_api_url}" return # Critical data integrity issue end - committer_date_str = commit_data_json.dig('commit', 'committer', 'date') + committer_date_str = commit_data_json.dig("commit", "committer", "date") unless committer_date_str Rails.logger.error "[ProcessCommitJob] Committer date not found in API response for commit #{commit_sha}. Data: #{commit_data_json.inspect}" return end - + begin # API dates are typically ISO8601 (UTC). Time.zone.parse respects the application's zone. # It's good practice to store in UTC, which parse will do correctly for ISO8601. @@ -91,11 +91,11 @@ class ProcessCommitJob < ApplicationJob elsif response.status.code == 404 Rails.logger.warn "[ProcessCommitJob] Commit #{commit_sha} not found (404) at #{commit_api_url} for User ##{user.id}." elsif response.status.code == 403 # Forbidden, could be rate limit or permissions - if response.headers['X-RateLimit-Remaining'].to_i == 0 - reset_time = Time.at(response.headers['X-RateLimit-Reset'].to_i) - delay_seconds = [(reset_time - Time.current).ceil, 5].max # at least 5s delay + if response.headers["X-RateLimit-Remaining"].to_i == 0 + reset_time = Time.at(response.headers["X-RateLimit-Reset"].to_i) + delay_seconds = [ (reset_time - Time.current).ceil, 5 ].max # at least 5s delay Rails.logger.warn "[ProcessCommitJob] GitHub API rate limit exceeded for User ##{user.id}. Retrying in #{delay_seconds}s. URL: #{commit_api_url}" - self.class.set(wait: delay_seconds.seconds).perform_later(user.id, commit_sha, commit_api_url, 'github') + self.class.set(wait: delay_seconds.seconds).perform_later(user.id, commit_sha, commit_api_url, "github") else Rails.logger.error "[ProcessCommitJob] GitHub API forbidden (403) for User ##{user.id}. URL: #{commit_api_url}. Response: #{response.body.to_s.truncate(500)}" end diff --git a/app/jobs/repo_host/sync_user_events_job.rb b/app/jobs/repo_host/sync_user_events_job.rb index 8abea9a..10085c3 100644 --- a/app/jobs/repo_host/sync_user_events_job.rb +++ b/app/jobs/repo_host/sync_user_events_job.rb @@ -1,20 +1,20 @@ -require 'http' # Make sure 'http' gem is available +require "http" # Make sure 'http' gem is available module RepoHost class SyncUserEventsJob < ApplicationJob queue_as :literally_whenever - + # MAX_API_PAGES_TO_FETCH: Max pages to fetch. GitHub's /users/{username}/events endpoint # is limited to 300 events. If per_page=100 (as we request), this is 3 pages. # If GitHub defaults to per_page=30, this would be 10 pages. # This constant acts as a safeguard. - MAX_API_PAGES_TO_FETCH = 10 + MAX_API_PAGES_TO_FETCH = 10 EVENTS_PER_PAGE = 100 discard_on ActiveJob::DeserializationError # Standard GoodJob practice # Retry with exponential backoff for transient network issues or temporary API errors - retry_on StandardError, wait: -> (executions) { [executions * 5, 60].min.seconds }, attempts: 3 + retry_on StandardError, wait: ->(executions) { [ executions * 5, 60 ].min.seconds }, attempts: 3 def perform(user_id:, provider:) @user = User.find_by(id: user_id) @@ -65,61 +65,61 @@ module RepoHost api_url = "#{base_api_url}&page=#{current_page}" Rails.logger.debug "Fetching GitHub events for User ##{@user.id}, Page #{current_page}, URL: #{api_url}" - + begin response = http_client_for_github.get(api_url) rescue HTTP::Error => e Rails.logger.error "RepoHost::SyncUserEventsJob: HTTP Error for User ##{@user.id} on page #{current_page}: #{e.message}" - break + break end unless response.status.success? handle_github_api_error(response, current_page) - break + break end fetched_events_json = response.parse Rails.logger.info "RepoHost::SyncUserEventsJob: User ##{@user.id}, Page #{current_page}: API returned #{fetched_events_json.size} events." - break if fetched_events_json.empty? + break if fetched_events_json.empty? events_to_create_on_this_page = [] stop_fetching_for_this_user = false fetched_events_json.each do |gh_event_data| - original_event_id_str = gh_event_data['id'].to_s + original_event_id_str = gh_event_data["id"].to_s repo_host_event_id = RepoHostEvent.construct_event_id(@provider_sym, original_event_id_str) - event_occurred_at = Time.zone.parse(gh_event_data['created_at']) + event_occurred_at = Time.zone.parse(gh_event_data["created_at"]) if latest_stored_event_db_created_at && event_occurred_at <= latest_stored_event_db_created_at if RepoHostEvent.exists?(id: repo_host_event_id, user_id: @user.id) Rails.logger.info "RepoHost::SyncUserEventsJob: Event ID #{repo_host_event_id} (occurred at #{event_occurred_at}) already exists for User ##{@user.id}. Stopping pagination." stop_fetching_for_this_user = true - break + break end end - + events_to_create_on_this_page << { id: repo_host_event_id, user_id: @user.id, raw_event_payload: gh_event_data, provider: RepoHostEvent.providers[@provider_sym], - created_at: event_occurred_at, - updated_at: Time.current + created_at: event_occurred_at, + updated_at: Time.current } - end + end if events_to_create_on_this_page.any? result = RepoHostEvent.import( events_to_create_on_this_page, - on_duplicate_key_ignore: { conflict_target: [:id] }, - validate: false + on_duplicate_key_ignore: { conflict_target: [ :id ] }, + validate: false ) newly_created_event_count_total += result.num_inserts Rails.logger.info "RepoHost::SyncUserEventsJob: For User ##{@user.id}, page #{current_page}: Processed #{events_to_create_on_this_page.size} events, imported #{result.num_inserts} new events." else Rails.logger.info "RepoHost::SyncUserEventsJob: For User ##{@user.id}, page #{current_page}: No new events to import." end - + break if stop_fetching_for_this_user # Manual pagination: increment page number for next request @@ -147,8 +147,8 @@ module RepoHost when 401 # Unauthorized Rails.logger.warn "GitHub token for User ##{@user.id} is likely invalid or expired. Sync aborted." when 403 # Forbidden - if response.headers['X-RateLimit-Remaining'].to_i == 0 - reset_time = Time.at(response.headers['X-RateLimit-Reset'].to_i) + if response.headers["X-RateLimit-Remaining"].to_i == 0 + reset_time = Time.at(response.headers["X-RateLimit-Reset"].to_i) Rails.logger.warn "GitHub API rate limit exceeded for User ##{@user.id}. Resets at #{reset_time}. Sync aborted." else Rails.logger.warn "GitHub API permission issue for User ##{@user.id} (e.g. fine-grained token scopes). Sync aborted." @@ -162,4 +162,4 @@ module RepoHost end end end -end \ No newline at end of file +end diff --git a/app/jobs/scan_repo_events_for_commits_job.rb b/app/jobs/scan_repo_events_for_commits_job.rb index ed1d2b1..75c69f2 100644 --- a/app/jobs/scan_repo_events_for_commits_job.rb +++ b/app/jobs/scan_repo_events_for_commits_job.rb @@ -21,11 +21,10 @@ class ScanRepoEventsForCommitsJob < ApplicationJob # Filter for GitHub PushEvents initially RepoHostEvent .where(provider: RepoHostEvent.providers[:github]) - .where("raw_event_payload->>'type' = ?", 'PushEvent') # Efficiently query JSONB + .where("raw_event_payload->>'type' = ?", "PushEvent") # Efficiently query JSONB .where("created_at >= ?", time_window_start) # Focus on recent events .order(created_at: :desc) # Process newer events first, potentially stopping earlier .find_each(batch_size: 100) do |event| - process_event(event) end @@ -43,7 +42,7 @@ class ScanRepoEventsForCommitsJob < ApplicationJob payload = event.raw_event_payload # Safely access nested commit data from the JSON payload - commits_data = payload.dig('payload', 'commits') + commits_data = payload.dig("payload", "commits") unless commits_data.is_a?(Array) && commits_data.any? # Rails.logger.debug "[ScanRepoEventsForCommitsJob] Event ID #{event.id} (User ##{user.id}) is a PushEvent but has no commits. Skipping." @@ -51,15 +50,15 @@ class ScanRepoEventsForCommitsJob < ApplicationJob end commits_data.each do |commit_info| - commit_sha = commit_info['sha'] + commit_sha = commit_info["sha"] # The 'url' in the PushEvent's commit object is the API URL for that commit - commit_api_url = commit_info['url'] + commit_api_url = commit_info["url"] if commit_sha.blank? || commit_api_url.blank? Rails.logger.warn "[ScanRepoEventsForCommitsJob] Event ID #{event.id} (User ##{user.id}) has a commit with missing SHA or API URL. Info: #{commit_info.inspect}" next end - + # Main check: Only enqueue if the commit SHA is not already in the Commit table. # This is crucial for idempotency and efficiency. unless Commit.exists?(sha: commit_sha) diff --git a/app/jobs/sync_all_user_repo_events_job.rb b/app/jobs/sync_all_user_repo_events_job.rb index 8508408..a31f1cf 100644 --- a/app/jobs/sync_all_user_repo_events_job.rb +++ b/app/jobs/sync_all_user_repo_events_job.rb @@ -3,7 +3,7 @@ class SyncAllUserRepoEventsJob < ApplicationJob def perform Rails.logger.info "Kicking off SyncAllUserRepoEventsJob" - + # Identify users: # 1. Authenticated with GitHub (have an access token and username) # 2. Have had heartbeats in the last 6 hours @@ -28,4 +28,4 @@ class SyncAllUserRepoEventsJob < ApplicationJob end Rails.logger.info "Successfully enqueued batch for GitHub event sync." end -end \ No newline at end of file +end diff --git a/app/jobs/update_airtable_user_data_job.rb b/app/jobs/update_airtable_user_data_job.rb index 06ff3d4..86c74cf 100644 --- a/app/jobs/update_airtable_user_data_job.rb +++ b/app/jobs/update_airtable_user_data_job.rb @@ -6,7 +6,7 @@ class UpdateAirtableUserDataJob < ApplicationJob def perform users_with_heartbeats.includes(:email_addresses).find_in_batches(batch_size: 100) do |batch| records = [] - + # Efficiently calculate total coding seconds for all users in this batch user_ids_in_batch = batch.map(&:id) total_coding_seconds_per_user = Heartbeat @@ -15,18 +15,18 @@ class UpdateAirtableUserDataJob < ApplicationJob .with_valid_timestamps .group(:user_id) # Group by user .duration_seconds # Returns a hash { user_id => seconds } - + batch.each do |user| first_heartbeat_time = user.heartbeats.with_valid_timestamps.order(time: :asc).limit(1).pluck(:time).first first_direct_heartbeat_time = user.heartbeats.direct_entry.with_valid_timestamps.order(time: :asc).limit(1).pluck(:time).first first_test_heartbeat_time = user.heartbeats.test_entry.with_valid_timestamps.order(time: :asc).limit(1).pluck(:time).first created_at = user.created_at.to_i next if first_heartbeat_time.nil? || first_heartbeat_time > Time.now.to_f - + # Get the pre-calculated total coding seconds for this user user_total_coding_seconds = total_coding_seconds_per_user[user.id] || 0 total_minutes_logged = (user_total_coding_seconds / 60).to_i - + user.email_addresses.map do |email_address| records << Table.new({ email: email_address.email, @@ -38,7 +38,7 @@ class UpdateAirtableUserDataJob < ApplicationJob }) end end - + # Only attempt to upsert if there are records to process Table.batch_upsert(records, "email") if records.any? end diff --git a/app/models/repo_host_event.rb b/app/models/repo_host_event.rb index b2965b5..96d7661 100644 --- a/app/models/repo_host_event.rb +++ b/app/models/repo_host_event.rb @@ -13,9 +13,9 @@ class RepoHostEvent < ApplicationRecord validates :created_at, presence: true # This is the event's occurrence time from the provider # Ensure ID starts with a recognized provider prefix - validates :id, format: { + validates :id, format: { with: /\A(gh|gl)_.+\z/, # Allow gh_ or gl_ prefixes - message: "must start with a provider prefix (e.g., gh_ or gl_)" + message: "must start with a provider prefix (e.g., gh_ or gl_)" } # Helper scope @@ -26,11 +26,11 @@ class RepoHostEvent < ApplicationRecord # Helper to construct the prefixed ID def self.construct_event_id(provider_name, original_event_id) prefix = case provider_name.to_sym - when :github then "gh_" - when :gitlab then "gl_" # Example for future - else + when :github then "gh_" + when :gitlab then "gl_" # Example for future + else raise ArgumentError, "Unknown provider: #{provider_name}" - end + end "#{prefix}#{original_event_id}" end end diff --git a/db/migrate/20250514180503_create_repo_host_events.rb b/db/migrate/20250514180503_create_repo_host_events.rb index 1138f62..1f939fd 100644 --- a/db/migrate/20250514180503_create_repo_host_events.rb +++ b/db/migrate/20250514180503_create_repo_host_events.rb @@ -19,6 +19,6 @@ class CreateRepoHostEvents < ActiveRecord::Migration[8.0] # Add an index for efficiently finding the latest event for a user/provider, # and for the "stop fetching if event exists" logic. # The primary key `id` is already unique and indexed. - add_index :repo_host_events, [:user_id, :provider, :created_at], name: 'index_repo_host_events_on_user_provider_created_at' + add_index :repo_host_events, [ :user_id, :provider, :created_at ], name: 'index_repo_host_events_on_user_provider_created_at' end end