Auto oss vs cost efficient 50/50 A/B test (#9355)

I want to run an a/b test where 50% of free users get defaulted to auto
cost efficient, and other 50% to auto open weights.

## Tests
- 50% of the time when i open the app with a different WARP_DATA_DIR i
get the expected flip/flopping
- my default gets persisted throughout signup

---------

Co-authored-by: Oz <oz-agent@warp.dev>

Author: IsaiahWitzke

app/src/ai/onboarding.rs

@@ -7,10 +7,15 @@ use warp_core::ui::icons::Icon;
 use warpui::{AppContext, SingletonEntity};
 
 use crate::auth::AuthStateProvider;
+use crate::experiments::FreeTierDefaultModel;
 use crate::workspaces::user_workspaces::UserWorkspaces;
 
 use super::llms::{DisableReason, LLMInfo, LLMPreferences};
 
+/// mirrors server-side model ids
+const AUTO_OPEN_LLM_ID: &str = "auto-open";
+const AUTO_COST_EFFICIENT_LLM_ID: &str = "auto-efficient";
+
 impl From<&LLMInfo> for OnboardingModelInfo {
     fn from(llm: &LLMInfo) -> Self {
         Self {
@@ -36,6 +41,27 @@ pub fn build_onboarding_models(prefs: &LLMPreferences) -> (Vec<OnboardingModelIn
     (models, default_id)
 }
 
+pub fn apply_free_tier_default_model_override(
+    models: &mut [OnboardingModelInfo],
+    server_default_id: LLMId,
+    ctx: &mut AppContext,
+) -> LLMId {
+    // server only gives back cost-efficient as a default if you're on a free or no plan
+    // if you ARE on some sort of plan... we should respect what the server says
+    if server_default_id != LLMId::from(AUTO_COST_EFFICIENT_LLM_ID) {
+        return server_default_id;
+    }
+    let auto_open_id = LLMId::from(AUTO_OPEN_LLM_ID);
+    let auto_open_available = models.iter().any(|m| m.id == auto_open_id);
+    if !auto_open_available || !FreeTierDefaultModel::should_default_to_auto_open(ctx) {
+        return server_default_id;
+    }
+    for m in models.iter_mut() {
+        m.is_default = m.id == auto_open_id;
+    }
+    auto_open_id
+}
+
 pub fn current_onboarding_auth_state(ctx: &AppContext) -> OnboardingAuthState {
     let auth_state = AuthStateProvider::as_ref(ctx).get();
     if auth_state.is_anonymous_or_logged_out() {

app/src/experiments/free_tier_default_model_layer.rs

@@ -0,0 +1,71 @@
+use super::{BucketRange, Experiment, Layer};
+use lazy_static::lazy_static;
+use std::collections::HashMap;
+use std::str::FromStr;
+use warpui::AppContext;
+
+lazy_static! {
+    pub static ref FREE_TIER_DEFAULT_MODEL_LAYER: Layer = Layer {
+        name: "FreeTierDefaultModelLayer",
+        hasher_seeds: (3141, 5926),
+        traffic_allocations: HashMap::from([
+            (FreeTierDefaultModel::AutoEfficient.get_group_id(), 50.0),
+            (FreeTierDefaultModel::AutoOpen.get_group_id(), 50.0),
+        ]),
+        bucket_ranges: vec![
+            BucketRange::new(FreeTierDefaultModel::AutoEfficient, 0..500),
+            BucketRange::new(FreeTierDefaultModel::AutoOpen, 500..1000),
+        ]
+    };
+}
+
+/// 50/50 A/B test of the default model surfaced to free-tier users in the
+/// pre-signup onboarding ("configure oz") model picker.
+#[derive(Debug)]
+pub enum FreeTierDefaultModel {
+    /// Control: keep the existing free-tier default (auto (cost-efficient)).
+    AutoEfficient,
+    /// Experiment: surface auto (open-weights) as the default for free users.
+    AutoOpen,
+}
+
+const FREE_TIER_DEFAULT_MODEL_AUTO_EFFICIENT: &str = "AutoEfficient";
+const FREE_TIER_DEFAULT_MODEL_AUTO_OPEN: &str = "AutoOpen";
+
+impl Experiment<FreeTierDefaultModel> for FreeTierDefaultModel {
+    fn name() -> &'static str {
+        "FreeTierDefaultModel"
+    }
+
+    fn variant(&self) -> &'static str {
+        match self {
+            Self::AutoEfficient => FREE_TIER_DEFAULT_MODEL_AUTO_EFFICIENT,
+            Self::AutoOpen => FREE_TIER_DEFAULT_MODEL_AUTO_OPEN,
+        }
+    }
+
+    fn allow_user_overrides_in_stable() -> bool {
+        false
+    }
+}
+
+impl FromStr for FreeTierDefaultModel {
+    type Err = anyhow::Error;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        match s {
+            FREE_TIER_DEFAULT_MODEL_AUTO_EFFICIENT => Ok(Self::AutoEfficient),
+            FREE_TIER_DEFAULT_MODEL_AUTO_OPEN => Ok(Self::AutoOpen),
+            _ => Err(anyhow::anyhow!(
+                "Variant {} is not a valid group in FreeTierDefaultModel",
+                s
+            )),
+        }
+    }
+}
+
+impl FreeTierDefaultModel {
+    pub fn should_default_to_auto_open(ctx: &mut AppContext) -> bool {
+        matches!(Self::get_group(ctx), Some(Self::AutoOpen))
+    }
+}

app/src/experiments/mod.rs

@@ -9,6 +9,7 @@ mod block_onboarding_layer;
 mod login_layer;
 mod rendering;
 pub use block_onboarding_layer::{BlockOnboarding, BLOCK_ONBOARDING_LAYER};
+pub use free_tier_default_model_layer::{FreeTierDefaultModel, FREE_TIER_DEFAULT_MODEL_LAYER};
 pub use improved_palette_search_layer::{ImprovedPaletteSearch, IMPROVED_PALETTE_SEARCH_LAYER};
 pub use login_layer::{AuthFlowInstructions, LOGIN_LAYER};
 use warp_core::user_preferences::GetUserPreferences as _;
@@ -69,6 +70,7 @@ lazy_static! {
         &*BLOCK_ONBOARDING_LAYER,
         &*rendering::LAYER,
         &*IMPROVED_PALETTE_SEARCH_LAYER,
+        &*FREE_TIER_DEFAULT_MODEL_LAYER,
     ];
 
     /// Mapping of experiments to their respective layers. The mappings are built up
@@ -403,6 +405,7 @@ pub fn init(ctx: &mut AppContext) {
 #[path = "mod_tests.rs"]
 mod tests;
 
+mod free_tier_default_model_layer;
 mod improved_palette_search_layer;
 #[cfg(test)]
 mod validation_tests;

app/src/root_view.rs

@@ -100,7 +100,9 @@ use warpui::keymap::{EditableBinding, FixedBinding};
 use warpui::windowing::WindowManager;
 
 use crate::ai::llms::{LLMPreferences, LLMPreferencesEvent};
-use crate::ai::onboarding::{build_onboarding_models, current_onboarding_auth_state};
+use crate::ai::onboarding::{
+    apply_free_tier_default_model_override, build_onboarding_models, current_onboarding_auth_state,
+};
 use crate::pricing::{PricingInfoModel, PricingInfoModelEvent};
 use warp_graphql::billing::StripeSubscriptionPlan;
 
@@ -1995,8 +1997,10 @@ impl RootView {
 
         let themes = onboarding_theme_picker_themes();
         let onboarding_view = ctx.add_typed_action_view(move |ctx| {
-            let llm_preferences = LLMPreferences::as_ref(ctx);
-            let (models, default_model_id) = build_onboarding_models(llm_preferences);
+            let (mut models, default_model_id) =
+                build_onboarding_models(LLMPreferences::as_ref(ctx));
+            let default_model_id =
+                apply_free_tier_default_model_override(&mut models, default_model_id, ctx);
 
             let workspace_enforces_autonomy = UserWorkspaces::as_ref(ctx)
                 .ai_autonomy_settings()
@@ -2039,8 +2043,10 @@ impl RootView {
             &LLMPreferences::handle(ctx),
             move |_, llm_preferences, event, ctx| match event {
                 LLMPreferencesEvent::UpdatedAvailableLLMs => {
-                    let (models, default_model_id) =
+                    let (mut models, default_model_id) =
                         build_onboarding_models(llm_preferences.as_ref(ctx));
+                    let default_model_id =
+                        apply_free_tier_default_model_override(&mut models, default_model_id, ctx);
                     onboarding_view_clone.update(ctx, |onboarding_view, ctx| {
                         onboarding_view.set_onboarding_models(models, default_model_id, ctx);
                     })