Benchmarking Updates (reflex-dev#3680)

* Benchmark test

* Lighthouse posthog

---------

Co-authored-by: Alek Petuskey <alekpetuskey@Aleks-MacBook-Pro.local>

Author: Alek99

.github/workflows/benchmarks.yml

@@ -18,7 +18,6 @@ env:
   PYTHONIOENCODING: 'utf8'
   TELEMETRY_ENABLED: false
   NODE_OPTIONS: '--max_old_space_size=4096'
-  DATABASE_URL: ${{ secrets.DATABASE_URL }}
   PR_TITLE: ${{ github.event.pull_request.title }}
 
 jobs:
@@ -62,17 +61,16 @@ jobs:
         run: |
           # Check that npm is home
           npm -v
-          poetry run bash scripts/benchmarks/benchmarks.sh ./reflex-web prod
+          poetry run bash benchmarks/lighthouse.sh ./reflex-web prod
         env:
           LHCI_GITHUB_APP_TOKEN: $
       - name: Run Benchmarks
         # Only run if the database creds are available in this context.
-        if: ${{ env.DATABASE_URL }}
-        run: poetry run python scripts/benchmarks/lighthouse_score_upload.py "$GITHUB_SHA" ./integration/benchmarks/.lighthouseci
+        run: poetry run python benchmarks/benchmark_lighthouse.py "$GITHUB_SHA" ./integration/benchmarks/.lighthouseci
         env:
           GITHUB_SHA: ${{ github.sha }}
 
-  simple-apps-benchmarks:
+  simple-apps-benchmarks: # This app tests the compile times of various compoonents and pages
     if: github.event.pull_request.merged == true
     env:
       OUTPUT_FILE: benchmarks.json
@@ -116,8 +114,6 @@ jobs:
           python-version: ${{ matrix.python-version }}
           run-poetry-install: true
           create-venv-at-path: .venv
-      - name: Install additional dependencies for DB access
-        run: poetry run uv pip install psycopg2-binary
       - name: Run benchmark tests
         env:
           APP_HARNESS_HEADLESS: 1
@@ -126,16 +122,14 @@ jobs:
           poetry run pytest -v benchmarks/ --benchmark-json=${{ env.OUTPUT_FILE }} -s
       - name: Upload benchmark results
         # Only run if the database creds are available in this context.
-        if: ${{ env.DATABASE_URL }}
         run:
-          poetry run python scripts/benchmarks/simple_app_benchmark_upload.py --os "${{ matrix.os }}"
+          poetry run python benchmarks/benchmark_compile_times.py --os "${{ matrix.os }}"
           --python-version "${{ matrix.python-version }}" --commit-sha "${{ github.sha }}"
-          --benchmark-json "${{ env.OUTPUT_FILE }}"
-          --db-url "${{ env.DATABASE_URL }}" --branch-name "${{ github.head_ref || github.ref_name }}"
-          --event-type "${{ github.event_name }}" --actor "${{ github.actor }}" --pr-id "${{ github.event.pull_request.id }}"
+          --benchmark-json "${{ env.OUTPUT_FILE }}" --branch-name "${{ github.head_ref || github.ref_name }}"
+          --event-type "${{ github.event_name }}" --pr-id "${{ github.event.pull_request.id }}"
 
-  reflex-build-size:
-    if: github.event.pull_request.merged == true
+  reflex-dist-size: # This job is used to calculate the size of the Reflex distribution (wheel file)
+    if: github.event.pull_request.merged == true  
     timeout-minutes: 30
     strategy:
       # Prioritize getting more information out of the workflow (even if something fails)
@@ -148,22 +142,19 @@ jobs:
           python-version: 3.11.5
           run-poetry-install: true
           create-venv-at-path: .venv
-      - name: Install additional dependencies for DB access
-        run: poetry run uv pip install psycopg2-binary
       - name: Build reflex
         run: |
           poetry build
       - name: Upload benchmark results
         # Only run if the database creds are available in this context.
-        if: ${{ env.DATABASE_URL }}
         run:
-          poetry run python scripts/benchmarks/benchmark_reflex_size.py --os ubuntu-latest
+          poetry run python benchmarks/benchmark_package_size.py --os ubuntu-latest
           --python-version 3.11.5 --commit-sha "${{ github.sha }}" --pr-id "${{ github.event.pull_request.id }}"
-          --db-url "${{ env.DATABASE_URL }}" --branch-name "${{ github.head_ref || github.ref_name }}"
-          --measurement-type "reflex-build" --path ./dist
+          --branch-name "${{ github.head_ref || github.ref_name }}"
+          --path ./dist
 
-  reflex-plus-dependency-size:
-    if: github.event.pull_request.merged == true
+  reflex-venv-size: # This job calculates the total size of Reflex and its dependencies
+    if: github.event.pull_request.merged == true  
     timeout-minutes: 30
     strategy:
       # Prioritize getting more information out of the workflow (even if something fails)
@@ -197,14 +188,10 @@ jobs:
         run: |
           poetry run pip install uv
 
-      - name: Install additional dependencies for DB access
-        run: poetry run uv pip install psycopg2-binary
-
-      - if: ${{ env.DATABASE_URL }}
-        name: calculate and upload size
+      - name: calculate and upload size
         run:
-          poetry run python scripts/benchmarks/benchmark_reflex_size.py --os "${{ matrix.os }}"
+          poetry run python benchmarks/benchmark_package_size.py --os "${{ matrix.os }}"
           --python-version "${{ matrix.python-version }}" --commit-sha "${{ github.sha }}"
-          --pr-id "${{ github.event.pull_request.id }}" --db-url "${{ env.DATABASE_URL }}"
+          --pr-id "${{ github.event.pull_request.id }}" 
           --branch-name "${{ github.head_ref || github.ref_name }}"
-          --measurement-type "reflex-package" --path ./.venv
+          --path ./.venv

.github/workflows/integration_tests.yml

@@ -30,7 +30,6 @@ env:
   PYTHONIOENCODING: 'utf8'
   TELEMETRY_ENABLED: false
   NODE_OPTIONS: '--max_old_space_size=4096'
-  DATABASE_URL: ${{ secrets.DATABASE_URL }}
   PR_TITLE: ${{ github.event.pull_request.title }}
 
 jobs:
@@ -100,27 +99,25 @@ jobs:
           npm -v
           poetry run bash scripts/integration.sh ./reflex-examples/counter dev
       - name: Measure and upload .web size
-        if: ${{ env.DATABASE_URL}}
         run:
-          poetry run python scripts/benchmarks/benchmark_reflex_size.py --os "${{ matrix.os }}"
+          poetry run python benchmarks/benchmark_web_size.py --os "${{ matrix.os }}"
           --python-version "${{ matrix.python-version }}" --commit-sha "${{ github.sha }}"
-          --pr-id "${{ github.event.pull_request.id }}" --db-url "${{ env.DATABASE_URL }}"
+          --pr-id "${{ github.event.pull_request.id }}"
           --branch-name "${{ github.head_ref || github.ref_name }}"
-          --measurement-type "counter-app-dot-web" --path ./reflex-examples/counter/.web
+          --path ./reflex-examples/counter/.web
+          --app-name "counter"
       - name: Install hyperfine
         run: cargo install hyperfine
       - name: Benchmark imports
         working-directory: ./reflex-examples/counter
         run: hyperfine --warmup 3 "export POETRY_VIRTUALENVS_PATH=../../.venv; poetry run python counter/counter.py" --show-output --export-json "${{ env.OUTPUT_FILE }}" --shell bash
       - name: Upload Benchmarks
-        if : ${{ env.DATABASE_URL }}
         run:
-          poetry run python scripts/benchmarks/benchmark_imports.py --os "${{ matrix.os }}"
+          poetry run python benchmarks/benchmark_imports.py --os "${{ matrix.os }}"
           --python-version "${{ matrix.python-version }}" --commit-sha "${{ github.sha }}"
           --benchmark-json "./reflex-examples/counter/${{ env.OUTPUT_FILE }}"
-          --db-url "${{ env.DATABASE_URL }}" --branch-name "${{ github.head_ref || github.ref_name }}"
-          --event-type "${{ github.event_name }}" --actor "${{ github.actor }}" --pr-id "${{ github.event.pull_request.id }}"
-
+          --branch-name "${{ github.head_ref || github.ref_name }}" --pr-id "${{ github.event.pull_request.id }}"
+          --app-name "counter"
 
 
 
@@ -164,10 +161,8 @@ jobs:
           npm -v
           poetry run bash scripts/integration.sh ./reflex-web prod
       - name: Measure and upload .web size
-        if: ${{ env.DATABASE_URL}}
         run:
-          poetry run python scripts/benchmarks/benchmark_reflex_size.py --os "${{ matrix.os }}"
+          poetry run python benchmarks/benchmark_web_size.py --os "${{ matrix.os }}"
           --python-version "${{ matrix.python-version }}" --commit-sha "${{ github.sha }}"
-          --pr-id "${{ github.event.pull_request.id }}"
-          --db-url "${{ env.DATABASE_URL }}" --branch-name "${{ github.head_ref || github.ref_name }}"
-          --measurement-type "reflex-web-dot-web" --path ./reflex-web/.web
+          --pr-id "${{ github.event.pull_request.id }}" --branch-name "${{ github.head_ref || github.ref_name }}"
+          --app-name "reflex-web" --path ./reflex-web/.web

…enchmarks/simple_app_benchmark_upload.py benchmarks/benchmark_compile_times.pyscripts/benchmarks/simple_app_benchmark_upload.py renamed to benchmarks/benchmark_compile_times.py scripts/benchmarks/simple_app_benchmark_upload.py renamed to benchmarks/benchmark_compile_times.py

@@ -1,13 +1,12 @@
-"""Runs the benchmarks and inserts the results into the database."""
+"""Extracts the compile times from the JSON files in the specified directory and inserts them into the database."""
 
 from __future__ import annotations
 
 import argparse
 import json
 import os
-from datetime import datetime
 
-import psycopg2
+from utils import send_data_to_posthog
 
 
 def extract_stats_from_json(json_file: str) -> list[dict]:
@@ -51,60 +50,40 @@ def extract_stats_from_json(json_file: str) -> list[dict]:
 
 
 def insert_benchmarking_data(
-    db_connection_url: str,
     os_type_version: str,
     python_version: str,
     performance_data: list[dict],
     commit_sha: str,
     pr_title: str,
     branch_name: str,
     event_type: str,
-    actor: str,
     pr_id: str,
 ):
     """Insert the benchmarking data into the database.
 
     Args:
-        db_connection_url: The URL to connect to the database.
         os_type_version: The OS type and version to insert.
         python_version: The Python version to insert.
         performance_data: The performance data of reflex web to insert.
         commit_sha: The commit SHA to insert.
         pr_title: The PR title to insert.
         branch_name: The name of the branch.
-        event_type: Type of github event(push, pull request, etc)
-        actor: Username of the user that triggered the run.
+        event_type: Type of github event(push, pull request, etc).
         pr_id: Id of the PR.
     """
-    # Serialize the JSON data
-    simple_app_performance_json = json.dumps(performance_data)
-
-    # Get the current timestamp
-    current_timestamp = datetime.now()
-
-    # Connect to the database and insert the data
-    with psycopg2.connect(db_connection_url) as conn, conn.cursor() as cursor:
-        insert_query = """
-            INSERT INTO simple_app_benchmarks (os, python_version, commit_sha, time, pr_title, branch_name, event_type, actor, performance, pr_id)
-            VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s);
-            """
-        cursor.execute(
-            insert_query,
-            (
-                os_type_version,
-                python_version,
-                commit_sha,
-                current_timestamp,
-                pr_title,
-                branch_name,
-                event_type,
-                actor,
-                simple_app_performance_json,
-                pr_id,
-            ),
-        )
-        # Commit the transaction
-        conn.commit()
+    # Prepare the event data
+    properties = {
+        "os": os_type_version,
+        "python_version": python_version,
+        "distinct_id": commit_sha,
+        "pr_title": pr_title,
+        "branch_name": branch_name,
+        "event_type": event_type,
+        "performance": performance_data,
+        "pr_id": pr_id,
+    }
+
+    send_data_to_posthog("simple_app_benchmark", properties)
 
 
 def main():
@@ -124,11 +103,6 @@ def main():
         "--benchmark-json",
         help="The JSON file containing the benchmark results.",
     )
-    parser.add_argument(
-        "--db-url",
-        help="The URL to connect to the database.",
-        required=True,
-    )
     parser.add_argument(
         "--pr-title",
         help="The PR title to insert into the database.",
@@ -143,11 +117,6 @@ def main():
         help="The github event type",
         required=True,
     )
-    parser.add_argument(
-        "--actor",
-        help="Username of the user that triggered the run.",
-        required=True,
-    )
     parser.add_argument(
         "--pr-id",
         help="ID of the PR.",
@@ -162,15 +131,13 @@ def main():
     cleaned_benchmark_results = extract_stats_from_json(args.benchmark_json)
     # Insert the data into the database
     insert_benchmarking_data(
-        db_connection_url=args.db_url,
         os_type_version=args.os,
         python_version=args.python_version,
         performance_data=cleaned_benchmark_results,
         commit_sha=args.commit_sha,
         pr_title=pr_title,
         branch_name=args.branch_name,
         event_type=args.event_type,
-        actor=args.actor,
         pr_id=args.pr_id,
     )