Merge ab-test-support into main

Author: teetow

.gitignore

@@ -34,5 +34,8 @@ pnpm-debug.log*
 # deno lock file (auto-generated)
 deno.lock
 
+# Local Netlify folder
+.netlify
+
 # vscode machine-specific settings
-.vscode/settings.json
+.vscode/

netlify.toml

@@ -0,0 +1,3 @@
+[[edge_functions]]
+  function = "cohort"
+  path = "/*"

netlify/edge-functions/cohort.ts

@@ -0,0 +1,24 @@
+import type { Context } from "https://edge.netlify.com";
+
+const COOKIE_NAME = "aud_ab_id";
+const MAX_ID = 100000;
+const ONE_YEAR = 60 * 60 * 24 * 365;
+
+export default async function cohort(request: Request, context: Context) {
+  const cookies = request.headers.get("cookie") ?? "";
+  const hasId = cookies
+    .split(";")
+    .some((c) => c.trim().startsWith(`${COOKIE_NAME}=`));
+
+  const response = await context.next();
+
+  if (!hasId) {
+    const id = Math.floor(Math.random() * MAX_ID);
+    response.headers.append(
+      "set-cookie",
+      `${COOKIE_NAME}=${id}; Path=/; Max-Age=${ONE_YEAR}; SameSite=Lax`,
+    );
+  }
+
+  return response;
+}

src/assets/data/experiments.ts

@@ -0,0 +1,40 @@
+export type Variant = {
+  name: string;
+  weight: number;
+};
+
+export type Experiment = {
+  name: string;
+  variants: Variant[];
+  enabled: boolean;
+};
+
+/**
+ * Register experiments here. Each experiment needs a unique name, two or more
+ * weighted variants, and an `enabled` flag.
+ *
+ * Example 50/50 test:
+ *
+ *   {
+ *     name: "hero-cta",
+ *     variants: [
+ *       { name: "control", weight: 50 },
+ *       { name: "variant-b", weight: 50 },
+ *     ],
+ *     enabled: true,
+ *   },
+ */
+export const experiments: Experiment[] = [
+  {
+    name: "nav-logo",
+    variants: [
+      { name: "control", weight: 50 },
+      { name: "text-only", weight: 50 },
+    ],
+    enabled: true,
+  },
+];
+
+export function getExperiment(name: string): Experiment | undefined {
+  return experiments.find((e) => e.name === name);
+}

src/assets/js/matomoTracking.js

@@ -1,3 +1,5 @@
+import { getAllAssignments, formatAssignments } from "../../utils/experiment";
+
 const getCookie = (name) => {
   const value = `; ${document.cookie}`;
   const parts = value.split(`; ${name}=`);
@@ -8,12 +10,15 @@ const branch = import.meta.env.BRANCH || "unknown-branch";
 
 var _paq = (window._paq = window._paq || []);
 /* tracker methods like "setCustomDimension" should be called before "trackPageView" */
-_paq.push(['setCustomDimension', 1, branch]); // ab-branch
+const assignments = getAllAssignments();
+const assignmentStr = formatAssignments(assignments);
+_paq.push(["setCustomDimension", 1, assignmentStr || branch]); // ab-branch
+
 _paq.push(["trackPageView"]);
 _paq.push(["enableLinkTracking"]);
 
 // Tell Matomo to wait for cookie consent
-_paq.push(['requireCookieConsent']);
+_paq.push(["requireCookieConsent"]);
 (function () {
   var u = "https://matomo.audacityteam.org/";
   _paq.push(["setTrackerUrl", u + "matomo.php"]);
@@ -28,5 +33,4 @@ _paq.push(['requireCookieConsent']);
 
 if (getCookie("audacity_consent") === "true") {
   _paq.push(["setCookieConsentGiven"]);
-} 
-
+}

src/components/navigation/NavigationReact.jsx

@@ -1,11 +1,13 @@
 import React, { useState } from "react";
 import AudacityLogo from "../../assets/img/Audacity_Logo.svg";
+import { useExperiment } from "../../hooks/useExperiment";
 import "@fontsource-variable/signika";
 import "../../styles/fonts.css";
 
 function NavigationReact(props) {
   const { currentURL } = props;
   const [isHamburgerMenuOpen, setIsHamburgerMenuOpen] = useState(false);
+  const { variant: navLogoVariant } = useExperiment("nav-logo");
 
   function getUrlPath(url) {
     const parts = url.split("/");
@@ -70,11 +72,13 @@ function NavigationReact(props) {
       <div className="flex h-14 items-center max-w-screen-2xl mx-auto px-4 md:px-6">
         <div className="flex-1">
           <a className="flex w-fit items-center gap-1 lg:gap-2" href="/">
-            <img
-              className="w-5 lg:w-6 h-full"
-              src={AudacityLogo.src}
-              alt="A yellow and orange waveform between the ears of a set of blue headphones"
-            />
+            {navLogoVariant !== "text-only" && (
+              <img
+                className="w-5 lg:w-6 h-full"
+                src={AudacityLogo.src}
+                alt="A yellow and orange waveform between the ears of a set of blue headphones"
+              />
+            )}
             <p className="signika text-blue-700 lg:text-lg font-medium lg:leading-none">
               Audacity
             </p>

src/hooks/useExperiment.ts

@@ -0,0 +1,33 @@
+import { useState, useEffect } from "react";
+import { getExperiment } from "../assets/data/experiments";
+import { getAbId, getVariant } from "../utils/experiment";
+
+type UseExperimentResult = {
+  variant: string | null;
+  isReady: boolean;
+};
+
+export function useExperiment(experimentName: string): UseExperimentResult {
+  const [result, setResult] = useState<UseExperimentResult>({
+    variant: null,
+    isReady: false,
+  });
+
+  useEffect(() => {
+    const experiment = getExperiment(experimentName);
+    if (!experiment || !experiment.enabled) {
+      setResult({ variant: null, isReady: true });
+      return;
+    }
+
+    const abId = getAbId();
+    if (abId === null) {
+      setResult({ variant: null, isReady: true });
+      return;
+    }
+
+    setResult({ variant: getVariant(experiment, abId), isReady: true });
+  }, [experimentName]);
+
+  return result;
+}

src/utils/experiment.test.ts

@@ -0,0 +1,87 @@
+import { describe, expect, test } from "bun:test";
+import { hashToSlot, getVariant } from "./experiment";
+import type { Experiment } from "../assets/data/experiments";
+
+describe("hashToSlot", () => {
+  test("returns a number between 0 and 99", () => {
+    for (let id = 0; id < 1000; id++) {
+      const slot = hashToSlot(id, "test-experiment");
+      expect(slot).toBeGreaterThanOrEqual(0);
+      expect(slot).toBeLessThan(100);
+    }
+  });
+
+  test("is deterministic for the same inputs", () => {
+    const a = hashToSlot(42, "hero-cta");
+    const b = hashToSlot(42, "hero-cta");
+    expect(a).toBe(b);
+  });
+
+  test("different experiment names produce different slots for same user", () => {
+    expect(hashToSlot(12345, "experiment-a")).toBe(83);
+    expect(hashToSlot(12345, "experiment-b")).toBe(82);
+  });
+
+  test("different user IDs produce different slots for same experiment", () => {
+    const slot1 = hashToSlot(1, "hero-cta");
+    const slot2 = hashToSlot(2, "hero-cta");
+    expect(slot1).not.toBe(slot2);
+  });
+});
+
+describe("getVariant", () => {
+  const fiftyFifty: Experiment = {
+    name: "test",
+    variants: [
+      { name: "control", weight: 50 },
+      { name: "variant-b", weight: 50 },
+    ],
+    enabled: true,
+  };
+
+  test("always returns a valid variant name", () => {
+    const validNames = fiftyFifty.variants.map((v) => v.name);
+    for (let id = 0; id < 1000; id++) {
+      const variant = getVariant(fiftyFifty, id);
+      expect(validNames).toContain(variant);
+    }
+  });
+
+  test("roughly 50/50 distribution over many IDs", () => {
+    const counts: Record<string, number> = { control: 0, "variant-b": 0 };
+    const total = 10000;
+    for (let id = 0; id < total; id++) {
+      counts[getVariant(fiftyFifty, id)]++;
+    }
+    // Allow 10% tolerance
+    expect(counts.control).toBeGreaterThan(total * 0.4);
+    expect(counts.control).toBeLessThan(total * 0.6);
+    expect(counts["variant-b"]).toBeGreaterThan(total * 0.4);
+    expect(counts["variant-b"]).toBeLessThan(total * 0.6);
+  });
+
+  test("respects unequal weights", () => {
+    const experiment: Experiment = {
+      name: "weighted",
+      variants: [
+        { name: "a", weight: 80 },
+        { name: "b", weight: 20 },
+      ],
+      enabled: true,
+    };
+    const counts: Record<string, number> = { a: 0, b: 0 };
+    const total = 10000;
+    for (let id = 0; id < total; id++) {
+      counts[getVariant(experiment, id)]++;
+    }
+    // "a" should get ~80% (allow 10% tolerance)
+    expect(counts.a).toBeGreaterThan(total * 0.7);
+    expect(counts.a).toBeLessThan(total * 0.9);
+  });
+
+  test("is deterministic", () => {
+    const v1 = getVariant(fiftyFifty, 42);
+    const v2 = getVariant(fiftyFifty, 42);
+    expect(v1).toBe(v2);
+  });
+});

src/utils/experiment.ts

@@ -0,0 +1,51 @@
+import { experiments, type Experiment } from "../assets/data/experiments";
+
+const COOKIE_NAME = "aud_ab_id";
+
+export function getAbId(): number | null {
+  if (typeof document === "undefined") return null;
+  const match = document.cookie
+    .split(";")
+    .map((c) => c.trim())
+    .find((c) => c.startsWith(`${COOKIE_NAME}=`));
+  if (!match) return null;
+  const val = parseInt(match.split("=")[1], 10);
+  return Number.isFinite(val) ? val : null;
+}
+
+/** Deterministic hash (djb2) of abId + experiment name → 0‑99 */
+export function hashToSlot(abId: number, experimentName: string): number {
+  const input = `${abId}:${experimentName}`;
+  let hash = 5381;
+  for (let i = 0; i < input.length; i++) {
+    hash = ((hash << 5) + hash + input.charCodeAt(i)) | 0;
+  }
+  return Math.abs(hash) % 100;
+}
+
+export function getVariant(experiment: Experiment, abId: number): string {
+  const slot = hashToSlot(abId, experiment.name);
+  let cumulative = 0;
+  for (const v of experiment.variants) {
+    cumulative += v.weight;
+    if (slot < cumulative) return v.name;
+  }
+  return experiment.variants[experiment.variants.length - 1].name;
+}
+
+export function getAllAssignments(): Record<string, string> {
+  const abId = getAbId();
+  if (abId === null) return {};
+  const result: Record<string, string> = {};
+  for (const exp of experiments) {
+    if (!exp.enabled) continue;
+    result[exp.name] = getVariant(exp, abId);
+  }
+  return result;
+}
+
+export function formatAssignments(assignments: Record<string, string>): string {
+  return Object.entries(assignments)
+    .map(([name, variant]) => `${name}:${variant}`)
+    .join("|");
+}