From 05da3a6b8408ae019dd48f54d6955fa79accf510 Mon Sep 17 00:00:00 2001 From: Echo Date: Wed, 25 Jun 2025 20:41:38 -0400 Subject: [PATCH] optimize mini leaderboards --- app/jobs/warm_mini_leaderboard_cache_job.rb | 20 +++++++++ app/models/concerns/heartbeatable.rb | 20 +++++++-- app/services/leaderboard_generator.rb | 35 ++++++++++----- .../leaderboards/_mini_leaderboard.html.erb | 44 +++++++++---------- config/initializers/cache_warming.rb | 11 +++++ ...add_indexes_for_leaderboard_performance.rb | 7 +++ db/schema.rb | 5 ++- 7 files changed, 103 insertions(+), 39 deletions(-) create mode 100644 app/jobs/warm_mini_leaderboard_cache_job.rb create mode 100644 config/initializers/cache_warming.rb create mode 100644 db/migrate/20250626001500_add_indexes_for_leaderboard_performance.rb diff --git a/app/jobs/warm_mini_leaderboard_cache_job.rb b/app/jobs/warm_mini_leaderboard_cache_job.rb new file mode 100644 index 0000000..28e5b0c --- /dev/null +++ b/app/jobs/warm_mini_leaderboard_cache_job.rb @@ -0,0 +1,20 @@ +class WarmMiniLeaderboardCacheJob < ApplicationJob + queue_as :default + + def perform + offsets = [ -8, -7, -6, -5, -4, -3, 0, 1, 2, 8, 9, 10, 11, 12 ] + + offsets.each do |offset| + begin + LeaderboardGenerator.generate_timezone_offset_leaderboard( + Date.current, + offset, + :daily + ) + Rails.logger.info "Warmed mini leaderboard cache for UTC#{offset >= 0 ? '+' : ''}#{offset}" + rescue => e + Rails.logger.error "Failed to warm cache for UTC#{offset >= 0 ? '+' : ''}#{offset}: #{e.message}" + end + end + end +end diff --git a/app/models/concerns/heartbeatable.rb b/app/models/concerns/heartbeatable.rb index 3b6a694..b85e585 100644 --- a/app/models/concerns/heartbeatable.rb +++ b/app/models/concerns/heartbeatable.rb @@ -98,9 +98,19 @@ module Heartbeatable end def daily_streaks_for_users(user_ids, start_date: 31.days.ago) - # First get the raw durations using window function + return {} if user_ids.empty? + start_date = [ start_date, 30.days.ago ].max + keys = user_ids.map { |id| "user_streak_#{id}" } + streak_cache = Rails.cache.read_multi(*keys) + + uncached_users = user_ids.select { |id| streak_cache["user_streak_#{id}"].nil? } + + if uncached_users.empty? + return user_ids.index_with { |id| streak_cache["user_streak_#{id}"] || 0 } + end + raw_durations = joins(:user) - .where(user_id: user_ids) + .where(user_id: uncached_users) .coding_only .with_valid_timestamps .where(time: start_date..Time.current) @@ -128,8 +138,7 @@ module Heartbeatable } end - # Initialize the result hash with zeros for all users - result = user_ids.index_with { 0 } + result = user_ids.index_with { |id| streak_cache["user_streak_#{id}"] || 0 } # Then calculate streaks for each user daily_durations.each do |user_id, data| @@ -158,6 +167,9 @@ module Heartbeatable end result[user_id] = streak + + # Cache the streak for 1 hour + Rails.cache.write("user_streak_#{user_id}", streak, expires_in: 1.hour) end result diff --git a/app/services/leaderboard_generator.rb b/app/services/leaderboard_generator.rb index 9ab4959..99a56f7 100644 --- a/app/services/leaderboard_generator.rb +++ b/app/services/leaderboard_generator.rb @@ -11,14 +11,24 @@ class LeaderboardGenerator def generate_timezone_offset_leaderboard(date, utc_offset, period_type = :daily) date = Date.current if date.blank? - users = User.users_in_timezone_offset(utc_offset).not_convicted - generate_leaderboard_for_users(users, date, "UTC#{utc_offset >= 0 ? '+' : ''}#{utc_offset}", period_type) + + cache_key = "timezone_leaderboard_#{utc_offset}_#{date}_#{period_type}" + + Rails.cache.fetch(cache_key, expires_in: 5.minutes) do + users = User.users_in_timezone_offset(utc_offset).not_convicted + generate_leaderboard_for_users(users, date, "UTC#{utc_offset >= 0 ? '+' : ''}#{utc_offset}", period_type) + end end def generate_timezone_leaderboard(date, timezone, period_type = :daily) date = Date.current if date.blank? - users = User.users_in_timezone(timezone).not_convicted - generate_leaderboard_for_users(users, date, timezone, period_type) + + cache_key = "timezone_leaderboard_#{timezone.gsub('/', '_')}_#{date}_#{period_type}" + + Rails.cache.fetch(cache_key, expires_in: 5.minutes) do + users = User.users_in_timezone(timezone).not_convicted + generate_leaderboard_for_users(users, date, timezone, period_type) + end end private @@ -34,10 +44,13 @@ class LeaderboardGenerator finished_generating_at: Time.current ) - # Get user IDs + # Get user IDs and preload users hash for faster lookups user_ids = users.pluck(:id) return leaderboard if user_ids.empty? + # Preload users into a hash for O(1) lookups + users_hash = users.index_by(&:id) + # Calculate heartbeats for the date range in UTC date_range = case period_type when :weekly @@ -48,7 +61,7 @@ class LeaderboardGenerator date.all_day end - # Get heartbeats for these users + # Get heartbeats for these users - limit to reduce query size heartbeats = Heartbeat.where(user_id: user_ids, time: date_range) .coding_only .with_valid_timestamps @@ -59,8 +72,10 @@ class LeaderboardGenerator user_totals = heartbeats.group(:user_id).duration_seconds user_totals = user_totals.filter { |_, total_seconds| total_seconds > 60 } - # Get streaks for all users at once - streaks = Heartbeat.daily_streaks_for_users(user_totals.keys) if user_totals.any? + # Only calculate streaks for users who actually have time today + # This significantly reduces the streak calculation overhead + streak_user_ids = user_totals.keys + streaks = streak_user_ids.any? ? Heartbeat.daily_streaks_for_users(streak_user_ids, start_date: 30.days.ago) : {} # Create virtual leaderboard entries entries = user_totals.map do |user_id, total_seconds| @@ -71,8 +86,8 @@ class LeaderboardGenerator streak_count: streaks[user_id] || 0 ) - # Manually set the user association to avoid N+1 queries - entry.user = users.find { |u| u.id == user_id } + # Use preloaded users hash instead of find + entry.user = users_hash[user_id] entry end.sort_by(&:total_seconds).reverse diff --git a/app/views/leaderboards/_mini_leaderboard.html.erb b/app/views/leaderboards/_mini_leaderboard.html.erb index 1328abb..f4d619e 100644 --- a/app/views/leaderboards/_mini_leaderboard.html.erb +++ b/app/views/leaderboards/_mini_leaderboard.html.erb @@ -2,23 +2,17 @@ <% if leaderboard.present? %> <% entries = leaderboard.entries.respond_to?(:order) ? leaderboard.entries.order(total_seconds: :desc) : leaderboard.entries - if current_user - user_rank = entries.find_index { |entry| entry.user_id == current_user.id } - if user_rank && user_rank >= 3 - # Show top 2 entries and immediate competition - top_entries = entries[0..1] - competition_entries = entries[[user_rank - 1, 2].max..[user_rank + 1, entries.size - 1].min] - mini_leaderboard_entries = top_entries + competition_entries - show_top_entries = false - else - # Show top 3 entries (either user is in top 3 or not on leaderboard) - mini_leaderboard_entries = entries.first(3) - show_top_entries = true - end + + top_entries = entries.first(3) + user_entry = current_user ? entries.find { |entry| entry.user_id == current_user.id } : nil + user_rank = user_entry ? entries.index(user_entry) : nil + + if user_entry && user_rank && user_rank >= 3 + mini_leaderboard_entries = top_entries + [user_entry] + show_user_separately = true else - # Not logged in, show top 3 - mini_leaderboard_entries = entries.first(3) - show_top_entries = true + mini_leaderboard_entries = top_entries + show_user_separately = false end %> @@ -36,20 +30,22 @@

<% mini_leaderboard_entries.each_with_index do |entry, idx| %> - <% is_competition = !show_top_entries && idx >= 2 %> + <% + actual_rank = entries.index(entry) + is_user_entry = show_user_separately && idx == 3 + %>
- <% if !is_competition %> - <% rank_emoji = case entries.index(entry) + <% if is_user_entry && actual_rank > 2 %> +
...
+
<%= actual_rank + 1 %>
+ <% else %> + <% rank_emoji = case actual_rank when 0 then "🥇" when 1 then "🥈" when 2 then "🥉" + else actual_rank + 1 end %>
<%= rank_emoji %>
- <% else %> - <% if idx == 2 && entries.index(entry) - entries.index(mini_leaderboard_entries[1]) > 1 %> -
...
- <% end %> -
<%= entries.index(entry) + 1 %>
<% end %>
diff --git a/config/initializers/cache_warming.rb b/config/initializers/cache_warming.rb new file mode 100644 index 0000000..32b664e --- /dev/null +++ b/config/initializers/cache_warming.rb @@ -0,0 +1,11 @@ +# Schedule mini leaderboard cache warming +# This will run every 5 minutes to keep the cache warm +Rails.application.config.after_initialize do + if defined?(Sidekiq::Cron::Job) + Sidekiq::Cron::Job.create( + name: "Warm Mini Leaderboard Cache", + cron: "*/5 * * * *", + class: "WarmMiniLeaderboardCacheJob" + ) + end +end diff --git a/db/migrate/20250626001500_add_indexes_for_leaderboard_performance.rb b/db/migrate/20250626001500_add_indexes_for_leaderboard_performance.rb new file mode 100644 index 0000000..c84a8cb --- /dev/null +++ b/db/migrate/20250626001500_add_indexes_for_leaderboard_performance.rb @@ -0,0 +1,7 @@ +class AddIndexesForLeaderboardPerformance < ActiveRecord::Migration[8.0] + def change + add_index :heartbeats, [ :user_id, :time, :category ], name: 'index_heartbeats_on_user_time_category' + add_index :users, [ :timezone, :trust_level ], name: 'index_users_on_timezone_trust_level' + add_index :users, :github_uid, name: 'index_users_on_github_uid' + end +end diff --git a/db/schema.rb b/db/schema.rb index 87c9594..d06484c 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[8.0].define(version: 2025_06_23_135342) do +ActiveRecord::Schema[8.0].define(version: 2025_06_26_001500) do create_schema "pganalyze" # These are extensions that must be enabled in order to support this database @@ -248,6 +248,7 @@ ActiveRecord::Schema[8.0].define(version: 2025_06_23_135342) do t.index ["fields_hash"], name: "index_heartbeats_on_fields_hash_when_not_deleted", unique: true, where: "(deleted_at IS NULL)" t.index ["raw_heartbeat_upload_id"], name: "index_heartbeats_on_raw_heartbeat_upload_id" t.index ["source_type", "time", "user_id", "project"], name: "index_heartbeats_on_source_type_time_user_project" + t.index ["user_id", "time", "category"], name: "index_heartbeats_on_user_time_category" t.index ["user_id", "time"], name: "idx_heartbeats_user_time_active", where: "(deleted_at IS NULL)" t.index ["user_id"], name: "index_heartbeats_on_user_id" end @@ -495,7 +496,9 @@ ActiveRecord::Schema[8.0].define(version: 2025_06_23_135342) do t.boolean "allow_public_stats_lookup", default: true, null: false t.boolean "default_timezone_leaderboard", default: true, null: false t.index ["github_uid", "github_access_token"], name: "index_users_on_github_uid_and_access_token" + t.index ["github_uid"], name: "index_users_on_github_uid" t.index ["slack_uid"], name: "index_users_on_slack_uid", unique: true + t.index ["timezone", "trust_level"], name: "index_users_on_timezone_trust_level" t.index ["timezone"], name: "index_users_on_timezone" end