chore: remove gitea ci workflows

.gitea/workflows/deploy.yaml

@@ -1,88 +0,0 @@
-name: Build, Test & Deploy LotusGen
-
-on:
-  push:
-    branches:
-      - main
-      - master
-  pull_request:
-    branches:
-      - main
-      - master
-
-jobs:
-  # -------------------------------------------------------------
-  # 1. Automated Testing (Backend Pytest + Frontend Typecheck)
-  # -------------------------------------------------------------
-  test:
-    name: Run Pytest & Frontend Build Test
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout Code
-        uses: actions/checkout@v4
-
-      # --- PYTHON SETUP ---
-      - name: Set up Python 3.11
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.11'
-
-      - name: Install Dependencies & Run Tests
-        run: |
-          python -m pip install --upgrade pip
-          pip install -r backend/requirements.txt pytest
-          python -m pytest backend/tests/ -v
-
-      # --- NODE SETUP ---
-      - name: Set up Node.js 20
-        uses: actions/setup-node@v4
-        with:
-          node-version: 20
-
-      - name: Verify Frontend Compilation
-        run: |
-          cd frontend
-          npm ci
-          npm run build
-
-  # -------------------------------------------------------------
-  # 2. Continuous Deployment (Build & Recreate Container)
-  # -------------------------------------------------------------
-  deploy:
-    name: Build & Deploy Container
-    needs: test
-    runs-on: ubuntu-latest
-    # Security: Deploy ONLY on pushes to main by the repository owner
-    if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/master') && github.repository == 'Sylphian/lotusgen'
-    steps:
-      - name: Checkout Code
-        uses: actions/checkout@v4
-
-      - name: Build and Start Production Container
-        env:
-          CONTAINER_NAME: ${{ secrets.CONTAINER_NAME }}
-          HOST_PORT: ${{ secrets.HOST_PORT }}
-          DATA_DIR: ${{ secrets.DATA_DIR }}
-          ARTIFACTS_DIR: ${{ secrets.ARTIFACTS_DIR }}
-        run: |
-          # Create host storage directories if they don't exist
-          mkdir -p "$DATA_DIR" "$ARTIFACTS_DIR"
-
-          # Build the production image from repo checkout
-          docker build -t lotusgen-app:latest .
-
-          # Gracefully recreate container
-          docker stop "$CONTAINER_NAME" 2>/dev/null || true
-          docker rm "$CONTAINER_NAME" 2>/dev/null || true
-
-          docker run -d \
-            --name "$CONTAINER_NAME" \
-            --restart unless-stopped \
-            -p 127.0.0.1:${HOST_PORT}:8000 \
-            -v "${DATA_DIR}:/app/data" \
-            -v "${ARTIFACTS_DIR}:/app/artifacts" \
-            -e LOTUSGEN_DB_PATH=/app/data/lotusgen.db \
-            -e LOTUSGEN_ARTIFACTS_DIR=/app/artifacts \
-            lotusgen-app:latest
-
-          echo "LotusGen successfully deployed to 127.0.0.1:${HOST_PORT}"

backend/api/routes.py

@@ -1,5 +1,6 @@
 import json
 import os
+import secrets
 from typing import List, Optional
 from fastapi import APIRouter, Depends, HTTPException, Query, Response, status
 from fastapi.responses import FileResponse
@@ -43,6 +44,7 @@ def format_table_response(table, project_tables) -> TableResponse:
         archetype=table.archetype,  # type: ignore
         status=table.status,  # type: ignore
         format=table.format,  # type: ignore
+        use_project_seed=getattr(table, "use_project_seed", True),
         row_limit=table.row_limit,
         exclude_if=table.exclude_if,
         fields=table.fields_data or [],
@@ -232,6 +234,7 @@ async def preview_table(table_id: str, count: int = Query(10, le=50), db: AsyncS
     proj = await ProjectService.get_project(db, table.project_id)
     proj_name = proj.name if proj else ""
     proj_seed = proj.seed if proj else 42
+    effective_seed = proj_seed if getattr(table, "use_project_seed", True) else secrets.randbelow(10**9)
 
     try:
         samples = generate_dry_run_sample(
@@ -239,7 +242,7 @@ async def preview_table(table_id: str, count: int = Query(10, le=50), db: AsyncS
             ref_tables,
             sample_size=count,
             project_name=proj_name,
-            seed=proj_seed,
+            seed=effective_seed,
         )
         return DryRunResponse(success=True, sample_rows=samples)
     except Exception as e:

backend/db/database.py

@@ -34,6 +34,8 @@ async def init_db() -> None:
             tab_cols = [r[1] for r in connection.exec_driver_sql("PRAGMA table_info(tables)").fetchall()]
             if "slug" not in tab_cols:
                 connection.exec_driver_sql("ALTER TABLE tables ADD COLUMN slug VARCHAR(255) DEFAULT ''")
+            if "use_project_seed" not in tab_cols:
+                connection.exec_driver_sql("ALTER TABLE tables ADD COLUMN use_project_seed BOOLEAN DEFAULT 1")
 
         await conn.run_sync(migrate_slugs)
 

backend/db/models.py

@@ -1,6 +1,6 @@
 import datetime
 from typing import Any, List, Optional
-from sqlalchemy import DateTime, ForeignKey, Integer, String, Text
+from sqlalchemy import Boolean, DateTime, ForeignKey, Integer, String, Text
 from sqlalchemy.dialects.sqlite import JSON
 from sqlalchemy.orm import Mapped, mapped_column, relationship
 from backend.db.database import Base
@@ -38,6 +38,7 @@ class TableModel(Base):
     archetype: Mapped[str] = mapped_column(String(50), nullable=False)  # record, aggregate, reference
     status: Mapped[str] = mapped_column(String(50), default="draft")  # draft, valid
     format: Mapped[str] = mapped_column(String(20), default="csv")  # csv, parquet
+    use_project_seed: Mapped[bool] = mapped_column(Boolean, default=True)
     row_limit: Mapped[Optional[int]] = mapped_column(Integer, nullable=True)
     exclude_if: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
 

backend/engine/samplers.py

@@ -77,11 +77,21 @@ def sample_weighted_without_replacement(
     return [item for _, item in keys[:chosen_size]]
 
 
+HEX_UPPERCASE = "0123456789ABCDEF"
+
+
 def sample_regex_mask(mask: str, rng: random.Random) -> str:
-    # '#' -> digit, '?' -> uppercase, '@' -> lowercase, '*' -> alphanumeric
+    # '#' -> digit, '?' -> uppercase, '@' -> lowercase, '*' -> alphanumeric, '%' -> hex uppercase
+    # '\' -> escape next character literally
     out = []
+    escape = False
     for char in mask:
-        if char == "#":
+        if escape:
+            out.append(char)
+            escape = False
+        elif char == "\\":
+            escape = True
+        elif char == "#":
             out.append(rng.choice(string.digits))
         elif char == "?":
             out.append(rng.choice(string.ascii_uppercase))
@@ -89,8 +99,12 @@ def sample_regex_mask(mask: str, rng: random.Random) -> str:
             out.append(rng.choice(string.ascii_lowercase))
         elif char == "*":
             out.append(rng.choice(string.ascii_letters + string.digits))
+        elif char == "%":
+            out.append(rng.choice(HEX_UPPERCASE))
         else:
             out.append(char)
+    if escape:
+        out.append("\\")
     return "".join(out)
 
 

backend/schemas/models.py

@@ -28,6 +28,7 @@ class TableCreate(BaseModel):
     slug: Optional[str] = None
     archetype: Literal["record", "aggregate", "reference", "collection"]
     format: Optional[Literal["csv", "parquet"]] = "csv"
+    use_project_seed: Optional[bool] = True
     row_limit: Optional[int] = 100000
     exclude_if: Optional[str] = None
     fields: Optional[List[FieldSchema]] = Field(default_factory=list)
@@ -44,6 +45,7 @@ class TableUpdate(BaseModel):
     name: Optional[str] = None
     slug: Optional[str] = None
     format: Optional[Literal["csv", "parquet"]] = None
+    use_project_seed: Optional[bool] = None
     row_limit: Optional[int] = None
     exclude_if: Optional[str] = None
     fields: Optional[List[FieldSchema]] = None
@@ -66,6 +68,7 @@ class TableResponse(BaseModel):
     archetype: Literal["record", "aggregate", "reference", "collection"]
     status: Literal["draft", "valid"]
     format: Literal["csv", "parquet"]
+    use_project_seed: bool = True
     row_limit: Optional[int] = None
     exclude_if: Optional[str] = None
     fields: List[FieldSchema] = Field(default_factory=list)
@@ -135,6 +138,7 @@ class ExportTableSchema(BaseModel):
     archetype: Literal["record", "aggregate", "reference", "collection"]
     status: Literal["draft", "valid"]
     format: Optional[Literal["csv", "parquet"]] = "csv"
+    use_project_seed: Optional[bool] = True
     row_limit: Optional[int] = None
     exclude_if: Optional[str] = None
     fields: Optional[List[FieldSchema]] = None

backend/services/generation_service.py

@@ -1,6 +1,7 @@
 import asyncio
 import datetime
 import os
+import secrets
 import time
 import uuid
 from typing import Any, AsyncGenerator, Dict, List, Optional
@@ -114,6 +115,7 @@ class GenerationService:
             project = await ProjectService.get_project(session, project_id)
             project_name = project.name if project else ""
             project_seed = project.seed if project else 42
+            effective_seed = project_seed if table_dict.get("use_project_seed", True) else secrets.randbelow(10**9)
 
         GenerationService._broadcast(
             job_id,
@@ -145,7 +147,7 @@ class GenerationService:
                 gen = TableGenerator(
                     table_dict,
                     ref_tables,
-                    seed=project_seed,
+                    seed=effective_seed,
                     chunk_size=50000,
                     project_name=project_name,
                     total_count=target_rows,
@@ -174,7 +176,7 @@ class GenerationService:
                 gen = TableGenerator(
                     table_dict,
                     ref_tables,
-                    seed=project_seed,
+                    seed=effective_seed,
                     chunk_size=50000,
                     project_name=project_name,
                     total_count=target_rows,

backend/services/project_service.py

@@ -128,6 +128,7 @@ class ProjectService:
                     "archetype": t.archetype,
                     "status": t.status,
                     "format": t.format,
+                    "use_project_seed": getattr(t, "use_project_seed", True),
                     "row_limit": t.row_limit,
                     "exclude_if": t.exclude_if,
                     "fields": t.fields_data or [],
@@ -178,6 +179,7 @@ class ProjectService:
                 archetype=t.get("archetype", "record"),
                 status=t.get("status", "draft"),
                 format=t.get("format", "csv"),
+                use_project_seed=t.get("use_project_seed", True),
                 row_limit=t.get("row_limit"),
                 exclude_if=t.get("exclude_if"),
                 fields_data=t.get("fields", []),

backend/services/table_service.py

@@ -1,4 +1,5 @@
 import re
+import secrets
 import uuid
 from typing import Any, Dict, List, Optional
 from sqlalchemy import select
@@ -88,6 +89,7 @@ def model_to_table_dict(t: TableModel) -> Dict[str, Any]:
         "archetype": t.archetype,
         "status": t.status,
         "format": t.format,
+        "use_project_seed": getattr(t, "use_project_seed", True),
         "row_limit": t.row_limit,
         "exclude_if": t.exclude_if,
         "fields": t.fields_data or [],
@@ -130,6 +132,7 @@ class TableService:
             archetype=data.archetype,
             status="draft",
             format=data.format or "csv",
+            use_project_seed=data.use_project_seed if data.use_project_seed is not None else True,
             row_limit=data.row_limit,
             exclude_if=clean_exclude,
             fields_data=clean_fields,
@@ -194,6 +197,8 @@ class TableService:
             table.slug = data.slug.strip()
         if data.format is not None:
             table.format = data.format
+        if data.use_project_seed is not None:
+            table.use_project_seed = data.use_project_seed
         if data.row_limit is not None:
             table.row_limit = data.row_limit
         if data.exclude_if is not None:
@@ -292,6 +297,7 @@ class TableService:
         project = proj_res.scalar_one_or_none()
         project_name = project.name if project else ""
         project_seed = project.seed if project else 42
+        effective_seed = project_seed if getattr(table, "use_project_seed", True) else secrets.randbelow(10**9)
 
         try:
             sample_rows = generate_dry_run_sample(
@@ -299,7 +305,7 @@ class TableService:
                 ref_tables,
                 sample_size=10,
                 project_name=project_name,
-                seed=project_seed,
+                seed=effective_seed,
             )
         except Exception as e:
             raise TableValidationError(f"Dry-run generation failed: {type(e).__name__} - {e}") from e

backend/tests/test_api.py

@@ -604,3 +604,80 @@ async def test_metadata_variables_api_workflow():
         assert len(val_rows) == 10
         assert val_rows[0]["tag"] == "TelemetryHub:metrics:1:2500:777"
         assert val_rows[9]["tag"] == "TelemetryHub:metrics:10:2500:777"
+
+
+@pytest.mark.asyncio
+async def test_table_use_project_seed_toggle():
+    await init_db()
+    transport = ASGITransport(app=app)
+    async with AsyncClient(transport=transport, base_url="http://test") as client:
+        # 1. Create project with seed=42
+        proj_slug = f"seed-test-{uuid.uuid4().hex[:6]}"
+        res_p = await client.post(
+            "/api/projects",
+            json={"name": "SeedTest", "slug": proj_slug, "seed": 42},
+        )
+        assert res_p.status_code == 201
+
+        # 2. Create table with use_project_seed=True (default)
+        res_t1 = await client.post(
+            f"/api/projects/{proj_slug}/tables",
+            json={
+                "name": "deterministic_table",
+                "archetype": "record",
+                "use_project_seed": True,
+                "fields": [
+                    {
+                        "name": "rand_val",
+                        "role": "randomized",
+                        "type": "integer",
+                        "config": {"min": 1, "max": 1000000},
+                    }
+                ],
+            },
+        )
+        assert res_t1.status_code == 201
+        t1_id = res_t1.json()["id"]
+        assert res_t1.json()["use_project_seed"] is True
+
+        # Previews on deterministic table must be identical
+        p1 = (await client.get(f"/api/tables/{t1_id}/preview?count=5")).json()["sample_rows"]
+        p2 = (await client.get(f"/api/tables/{t1_id}/preview?count=5")).json()["sample_rows"]
+        assert p1 == p2
+
+        # 3. Create table with use_project_seed=False (dynamic seed)
+        res_t2 = await client.post(
+            f"/api/projects/{proj_slug}/tables",
+            json={
+                "name": "dynamic_table",
+                "archetype": "record",
+                "use_project_seed": False,
+                "fields": [
+                    {
+                        "name": "rand_val",
+                        "role": "randomized",
+                        "type": "integer",
+                        "config": {"min": 1, "max": 1000000},
+                    }
+                ],
+            },
+        )
+        assert res_t2.status_code == 201
+        t2_id = res_t2.json()["id"]
+        assert res_t2.json()["use_project_seed"] is False
+
+        # Previews on dynamic table will generate fresh seeds
+        dp1 = (await client.get(f"/api/tables/{t2_id}/preview?count=5")).json()["sample_rows"]
+        dp2 = (await client.get(f"/api/tables/{t2_id}/preview?count=5")).json()["sample_rows"]
+        assert dp1 != dp2
+
+        # 4. Update use_project_seed toggle via PUT /api/tables/{id}
+        res_up = await client.put(f"/api/tables/{t2_id}", json={"use_project_seed": True})
+        assert res_up.status_code == 200
+        assert res_up.json()["use_project_seed"] is True
+
+        # Now it is deterministic
+        up1 = (await client.get(f"/api/tables/{t2_id}/preview?count=5")).json()["sample_rows"]
+        up2 = (await client.get(f"/api/tables/{t2_id}/preview?count=5")).json()["sample_rows"]
+        assert up1 == up2
+

backend/tests/test_engine.py

@@ -19,7 +19,7 @@ from backend.engine.sandbox import (
     FormulaSyntaxError,
     extract_variable_names,
 )
-from backend.engine.samplers import generate_uuid7, get_random_sublist
+from backend.engine.samplers import generate_uuid7, get_random_sublist, sample_regex_mask
 from backend.engine.writers import StreamingCsvWriter, StreamingParquetWriter
 
 
@@ -657,3 +657,62 @@ def test_extended_builtin_metadata_variables():
     df_agg = pl.concat(chunks_agg)
     assert len(df_agg) == 6
     assert df_agg["meta_calc"].to_list() == ["1/6", "2/6", "3/6", "4/6", "5/6", "6/6"]
+
+
+def test_sample_regex_mask_hex_and_escaping():
+    import random
+    rng = random.Random(42)
+
+    # 1. Hexadecimal token '%' (0-9, A-F)
+    hex_sample = sample_regex_mask("0x%%%%", rng)
+    assert hex_sample.startswith("0x")
+    assert len(hex_sample) == 6
+    for c in hex_sample[2:]:
+        assert c in "0123456789ABCDEF"
+
+    # 2. Escaped special tokens '\#', '\?', '\@', '\*', '\%', '\H'
+    escaped_sample = sample_regex_mask(r"\#\?\@\*\%\Hello", rng)
+    assert escaped_sample == "#?@*%Hello"
+
+    # 3. Escaped backslash '\\'
+    path_sample = sample_regex_mask(r"C:\\DIR\\\#-####", rng)
+    assert path_sample.startswith(r"C:\DIR\#-")
+    assert len(path_sample) == 13
+    assert path_sample[-4:].isdigit()
+
+    # 4. Trailing backslash
+    trailing_sample = sample_regex_mask("TEST-\\\\", rng)
+    assert trailing_sample == "TEST-\\"
+
+    single_trailing = sample_regex_mask("TEST-\\", rng)
+    assert single_trailing == "TEST-\\"
+
+    # 5. Table generator end-to-end integration test with masked string
+    table = {
+        "name": "hex_table",
+        "archetype": "record",
+        "status": "valid",
+        "fields": [
+            {
+                "name": "mac_addr",
+                "role": "randomized",
+                "type": "string",
+                "config": {"mask": r"MAC-\#%%-\#%%-\#%%"},
+                "active": True,
+            }
+        ],
+    }
+    gen = TableGenerator(table, [], seed=77, chunk_size=5)
+    df = next(gen.generate_record_chunks(5))
+    macs = df["mac_addr"].to_list()
+    assert len(macs) == 5
+    for m in macs:
+        assert m.startswith("MAC-#")
+        parts = m.split("-")
+        assert len(parts) == 4
+        assert parts[0] == "MAC"
+        for p in parts[1:]:
+            assert p.startswith("#")
+            assert len(p) == 3
+            assert all(c in "0123456789ABCDEF" for c in p[1:])
+

frontend/src/components/TableSchemaEditor.tsx

@@ -1,4 +1,4 @@
-import React, { useState } from 'react';
+import React, { useEffect, useRef, useState } from 'react';
 import {
   ArrowLeft,
   ArrowUp,
@@ -14,6 +14,8 @@ import {
   Clipboard,
   Check,
   X,
+  BracketsCurly,
+  MagnifyingGlass,
 } from '@phosphor-icons/react';
 import { updateTable, validateTable, previewTable } from '../api';
 import type { ColumnRole, ColumnType, Field, Table } from '../types';
@@ -27,6 +29,16 @@ interface TableSchemaEditorProps {
   onGenerate: (format: 'csv' | 'parquet') => void;
 }
 
+const BUILTIN_METADATA_VARIABLES = [
+  { name: '_row', desc: '1-based row counter' },
+  { name: '_index', desc: '0-based row counter' },
+  { name: '_count', desc: 'Total target rows' },
+  { name: '_name', desc: 'Current table name' },
+  { name: '_project', desc: 'Project name' },
+  { name: '_timestamp', desc: 'UTC ISO timestamp' },
+  { name: '_seed', desc: 'Project RNG seed' },
+];
+
 export const TableSchemaEditor: React.FC<TableSchemaEditorProps> = ({
   table,
   allProjectTables,
@@ -36,6 +48,7 @@ export const TableSchemaEditor: React.FC<TableSchemaEditorProps> = ({
 }) => {
   const [name, setName] = useState(table.name);
   const [format, setFormat] = useState<'csv' | 'parquet'>(table.format || 'csv');
+  const [useProjectSeed, setUseProjectSeed] = useState<boolean>(table.use_project_seed !== false);
   const [rowLimit, setRowLimit] = useState<number>(table.row_limit || 100000);
   const [excludeIf, setExcludeIf] = useState<string>(table.exclude_if || '');
   const [fields, setFields] = useState<Field[]>(table.fields || []);
@@ -59,12 +72,45 @@ export const TableSchemaEditor: React.FC<TableSchemaEditorProps> = ({
   const [copied, setCopied] = useState(false);
   const [showConfirmGuard, setShowConfirmGuard] = useState(false);
 
+  // Variable Picker Popover state
+  const [activeVarPickerIdx, setActiveVarPickerIdx] = useState<number | null>(null);
+  const [varSearchQuery, setVarSearchQuery] = useState('');
+  const varPickerRef = useRef<HTMLDivElement>(null);
+
+  useEffect(() => {
+    if (activeVarPickerIdx === null) return;
+    const handleClickOutside = (e: MouseEvent) => {
+      if (varPickerRef.current && !varPickerRef.current.contains(e.target as Node)) {
+        setActiveVarPickerIdx(null);
+      }
+    };
+    const handleKeyDown = (e: KeyboardEvent) => {
+      if (e.key === 'Escape') {
+        setActiveVarPickerIdx(null);
+      }
+    };
+    document.addEventListener('mousedown', handleClickOutside);
+    document.addEventListener('keydown', handleKeyDown);
+    return () => {
+      document.removeEventListener('mousedown', handleClickOutside);
+      document.removeEventListener('keydown', handleKeyDown);
+    };
+  }, [activeVarPickerIdx]);
+
+  const handleInsertVariable = (fieldIdx: number, varName: string) => {
+    const currentExpr = fields[fieldIdx]?.config?.expression || '';
+    handleConfigChange(fieldIdx, 'expression', currentExpr + varName);
+    setActiveVarPickerIdx(null);
+    setVarSearchQuery('');
+  };
+
   const isLocked = table.status === 'valid' && table.referencing_tables.length > 0;
   const isAggregate = table.archetype === 'aggregate';
 
   const exportPayload = {
     archetype: table.archetype,
     format,
+    use_project_seed: useProjectSeed,
     row_limit: rowLimit,
     exclude_if: excludeIf || null,
     fields,
@@ -127,6 +173,9 @@ export const TableSchemaEditor: React.FC<TableSchemaEditorProps> = ({
         if (parsed.format && (parsed.format === 'csv' || parsed.format === 'parquet')) {
           setFormat(parsed.format);
         }
+        if (typeof parsed.use_project_seed === 'boolean') {
+          setUseProjectSeed(parsed.use_project_seed);
+        }
         if (typeof parsed.row_limit === 'number') {
           setRowLimit(parsed.row_limit);
         }
@@ -311,6 +360,7 @@ export const TableSchemaEditor: React.FC<TableSchemaEditorProps> = ({
       const updated = await updateTable(table.id, {
         name: cleanName,
         format,
+        use_project_seed: useProjectSeed,
         row_limit: rowLimit,
         exclude_if: cleanExclude,
         fields: cleanFields,
@@ -338,6 +388,7 @@ export const TableSchemaEditor: React.FC<TableSchemaEditorProps> = ({
       await updateTable(table.id, {
         name: cleanName,
         format,
+        use_project_seed: useProjectSeed,
         row_limit: rowLimit,
         exclude_if: cleanExclude,
         fields: cleanFields,
@@ -366,6 +417,7 @@ export const TableSchemaEditor: React.FC<TableSchemaEditorProps> = ({
       await updateTable(table.id, {
         name,
         format,
+        use_project_seed: useProjectSeed,
         row_limit: rowLimit,
         exclude_if: excludeIf || null,
         fields,
@@ -523,6 +575,32 @@ export const TableSchemaEditor: React.FC<TableSchemaEditorProps> = ({
             />
           </div>
 
+          <div>
+            <label className="block text-xs font-medium text-[#4F6170] dark:text-[#ADBFCE] mb-1">
+              RNG Seed
+            </label>
+            <button
+              type="button"
+              disabled={isLocked}
+              onClick={() => setUseProjectSeed(!useProjectSeed)}
+              className={`w-full flex items-center justify-between px-3 py-1.5 text-xs font-mono rounded-[6px] border-[1.5px] transition-colors disabled:opacity-50 cursor-pointer ${
+                useProjectSeed
+                  ? 'border-[#346538] bg-[#EDF3EC] dark:bg-[#1b2b1e] text-[#346538] dark:text-[#88d48e]'
+                  : 'border-[#DEE3E8] dark:border-[#4F6170] bg-[#FFFFFF] dark:bg-[#161C22] text-[#4F6170] dark:text-[#ADBFCE]'
+              }`}
+              title={
+                useProjectSeed
+                  ? 'Follows project seed (deterministic and reproducible)'
+                  : 'Uses dynamic random seed per generation/preview run'
+              }
+            >
+              <span>{useProjectSeed ? 'Project Seed' : 'Dynamic Seed'}</span>
+              <span className="text-[10px] font-bold uppercase px-1.5 py-0.5 rounded border border-current">
+                {useProjectSeed ? 'Fixed' : 'Random'}
+              </span>
+            </button>
+          </div>
+
           {isAggregate && (
             <div>
               <label className="block text-xs font-medium text-[#4F6170] dark:text-[#ADBFCE] mb-1">
@@ -690,102 +768,159 @@ export const TableSchemaEditor: React.FC<TableSchemaEditorProps> = ({
                         <span className="font-medium text-[#000000] dark:text-[#FFFFFF]">
                           Python Expression (Safe Sandbox with f-strings, math, random):
                         </span>
-                        <div className="flex flex-wrap items-center gap-1.5 text-[11px] font-mono text-[#4F6170] dark:text-[#ADBFCE]">
-                          <span>Built-ins:</span>
+
+                        {/* Unified Variable Picker Button & Popover */}
+                        <div className="relative">
                           <button
                             type="button"
-                            onClick={() => {
-                              const currentExpr = field.config?.expression || '';
-                              handleConfigChange(idx, 'expression', currentExpr + '_row');
+                            disabled={isLocked || !isActive}
+                            onClick={(e) => {
+                              e.stopPropagation();
+                              if (activeVarPickerIdx === idx) {
+                                setActiveVarPickerIdx(null);
+                              } else {
+                                setActiveVarPickerIdx(idx);
+                                setVarSearchQuery('');
+                              }
                             }}
-                            className="px-2 py-0.5 rounded bg-[#EDF3EC] dark:bg-[#1b2b1e] border-[1.5px] border-[#DEE3E8] dark:border-[#4F6170] hover:border-[#000000] dark:hover:border-[#FFFFFF] text-[#346538] dark:text-[#88d48e] transition-colors"
-                            title="1-based row counter (1, 2, 3...)"
+                            className={`flex items-center space-x-1.5 text-xs font-mono font-medium px-2.5 py-1 rounded-[5px] border-[1.5px] transition-colors disabled:opacity-40 cursor-pointer ${
+                              activeVarPickerIdx === idx
+                                ? 'border-[#000000] dark:border-[#FFFFFF] bg-[#EDF3EC] dark:bg-[#1b2b1e] text-[#346538] dark:text-[#88d48e]'
+                                : 'border-[#DEE3E8] dark:border-[#4F6170] bg-[#FFFFFF] dark:bg-[#0E1317] hover:border-[#000000] dark:hover:border-[#FFFFFF] text-[#000000] dark:text-[#FFFFFF]'
+                            }`}
+                            title="Insert built-in variable or upstream column into expression"
                           >
-                            _row
+                            <BracketsCurly size={13} weight="bold" />
+                            <span>Insert Variable</span>
                           </button>
-                          <button
-                            type="button"
-                            onClick={() => {
-                              const currentExpr = field.config?.expression || '';
-                              handleConfigChange(idx, 'expression', currentExpr + '_index');
-                            }}
-                            className="px-2 py-0.5 rounded bg-[#EDF3EC] dark:bg-[#1b2b1e] border-[1.5px] border-[#DEE3E8] dark:border-[#4F6170] hover:border-[#000000] dark:hover:border-[#FFFFFF] text-[#346538] dark:text-[#88d48e] transition-colors"
-                            title="0-based row counter (0, 1, 2...)"
+
+                          {activeVarPickerIdx === idx && (
+                            <div
+                              ref={varPickerRef}
+                              onClick={(e) => e.stopPropagation()}
+                              className="absolute right-0 top-full mt-1.5 w-80 max-h-[380px] overflow-y-auto z-40 bg-[#FFFFFF] dark:bg-[#0E1317] border-[1.5px] border-[#DEE3E8] dark:border-[#4F6170] rounded-[8px] shadow-xl p-3 space-y-3"
                             >
-                            _index
-                          </button>
-                          <button
-                            type="button"
-                            onClick={() => {
-                              const currentExpr = field.config?.expression || '';
-                              handleConfigChange(idx, 'expression', currentExpr + '_count');
-                            }}
-                            className="px-2 py-0.5 rounded bg-[#EDF3EC] dark:bg-[#1b2b1e] border-[1.5px] border-[#DEE3E8] dark:border-[#4F6170] hover:border-[#000000] dark:hover:border-[#FFFFFF] text-[#346538] dark:text-[#88d48e] transition-colors"
-                            title="Total rows target count (int)"
-                          >
-                            _count
-                          </button>
-                          <button
-                            type="button"
-                            onClick={() => {
-                              const currentExpr = field.config?.expression || '';
-                              handleConfigChange(idx, 'expression', currentExpr + '_name');
-                            }}
-                            className="px-2 py-0.5 rounded bg-[#EDF3EC] dark:bg-[#1b2b1e] border-[1.5px] border-[#DEE3E8] dark:border-[#4F6170] hover:border-[#000000] dark:hover:border-[#FFFFFF] text-[#346538] dark:text-[#88d48e] transition-colors"
-                            title="Current table name (str)"
-                          >
-                            _name
-                          </button>
-                          <button
-                            type="button"
-                            onClick={() => {
-                              const currentExpr = field.config?.expression || '';
-                              handleConfigChange(idx, 'expression', currentExpr + '_project');
-                            }}
-                            className="px-2 py-0.5 rounded bg-[#EDF3EC] dark:bg-[#1b2b1e] border-[1.5px] border-[#DEE3E8] dark:border-[#4F6170] hover:border-[#000000] dark:hover:border-[#FFFFFF] text-[#346538] dark:text-[#88d48e] transition-colors"
-                            title="Project name (str)"
-                          >
-                            _project
-                          </button>
-                          <button
-                            type="button"
-                            onClick={() => {
-                              const currentExpr = field.config?.expression || '';
-                              handleConfigChange(idx, 'expression', currentExpr + '_timestamp');
-                            }}
-                            className="px-2 py-0.5 rounded bg-[#EDF3EC] dark:bg-[#1b2b1e] border-[1.5px] border-[#DEE3E8] dark:border-[#4F6170] hover:border-[#000000] dark:hover:border-[#FFFFFF] text-[#346538] dark:text-[#88d48e] transition-colors"
-                            title="UTC generation timestamp in ISO 8601 (str)"
-                          >
-                            _timestamp
-                          </button>
-                          <button
-                            type="button"
-                            onClick={() => {
-                              const currentExpr = field.config?.expression || '';
-                              handleConfigChange(idx, 'expression', currentExpr + '_seed');
-                            }}
-                            className="px-2 py-0.5 rounded bg-[#EDF3EC] dark:bg-[#1b2b1e] border-[1.5px] border-[#DEE3E8] dark:border-[#4F6170] hover:border-[#000000] dark:hover:border-[#FFFFFF] text-[#346538] dark:text-[#88d48e] transition-colors"
-                            title="Project RNG seed (int)"
-                          >
-                            _seed
-                          </button>
-                          <span className="ml-1">Upstream:</span>
-                          {upstreamColumns.length === 0 ? (
-                            <span className="italic">none</span>
+                              {/* Search Input */}
+                              <div className="relative">
+                                <MagnifyingGlass
+                                  size={13}
+                                  className="absolute left-2.5 top-1/2 -translate-y-1/2 text-[#4F6170] dark:text-[#ADBFCE]"
+                                />
+                                <input
+                                  type="text"
+                                  autoFocus
+                                  value={varSearchQuery}
+                                  onChange={(e) => setVarSearchQuery(e.target.value)}
+                                  placeholder="Filter variables..."
+                                  className="w-full pl-8 pr-2.5 py-1.5 text-xs font-mono border-[1.5px] border-[#DEE3E8] dark:border-[#4F6170] rounded-[5px] bg-[#F4FAFF] dark:bg-[#161C22] text-[#000000] dark:text-[#FFFFFF] focus:outline-none focus:border-[#000000] dark:focus:border-[#FFFFFF]"
+                                />
+                              </div>
+
+                              {/* Fundamental / Built-in Variables Section */}
+                              <div className="space-y-1.5">
+                                <div className="text-[10px] font-mono uppercase tracking-wider font-semibold text-[#4F6170] dark:text-[#ADBFCE] flex items-center justify-between">
+                                  <span>Fundamental Variables</span>
+                                  <span>
+                                    {
+                                      BUILTIN_METADATA_VARIABLES.filter(
+                                        (v) =>
+                                          v.name
+                                            .toLowerCase()
+                                            .includes(varSearchQuery.trim().toLowerCase()) ||
+                                          v.desc
+                                            .toLowerCase()
+                                            .includes(varSearchQuery.trim().toLowerCase())
+                                      ).length
+                                    }
+                                  </span>
+                                </div>
+                                {BUILTIN_METADATA_VARIABLES.filter(
+                                  (v) =>
+                                    v.name
+                                      .toLowerCase()
+                                      .includes(varSearchQuery.trim().toLowerCase()) ||
+                                    v.desc
+                                      .toLowerCase()
+                                      .includes(varSearchQuery.trim().toLowerCase())
+                                ).length === 0 ? (
+                                  <div className="text-[11px] font-mono text-[#4F6170] dark:text-[#ADBFCE] italic px-1">
+                                    No matching variables
+                                  </div>
                                 ) : (
-                            upstreamColumns.map((col) => (
+                                  <div className="grid grid-cols-1 gap-1">
+                                    {BUILTIN_METADATA_VARIABLES.filter(
+                                      (v) =>
+                                        v.name
+                                          .toLowerCase()
+                                          .includes(varSearchQuery.trim().toLowerCase()) ||
+                                        v.desc
+                                          .toLowerCase()
+                                          .includes(varSearchQuery.trim().toLowerCase())
+                                    ).map((v) => (
+                                      <button
+                                        key={v.name}
+                                        type="button"
+                                        onClick={() => handleInsertVariable(idx, v.name)}
+                                        className="flex items-center justify-between text-left px-2 py-1.5 rounded-[5px] bg-[#EDF3EC] dark:bg-[#1b2b1e] border border-[#DEE3E8] dark:border-[#4F6170] hover:border-[#000000] dark:hover:border-[#FFFFFF] transition-colors group cursor-pointer"
+                                      >
+                                        <span className="font-mono text-xs font-bold text-[#346538] dark:text-[#88d48e]">
+                                          {v.name}
+                                        </span>
+                                        <span className="font-mono text-[10px] text-[#4F6170] dark:text-[#ADBFCE] group-hover:text-[#000000] dark:group-hover:text-[#FFFFFF] transition-colors truncate max-w-[155px]">
+                                          {v.desc}
+                                        </span>
+                                      </button>
+                                    ))}
+                                  </div>
+                                )}
+                              </div>
+
+                              {/* Upstream Columns Section */}
+                              <div className="space-y-1.5 border-t-[1.5px] border-[#DEE3E8] dark:border-[#4F6170]/60 pt-2">
+                                <div className="text-[10px] font-mono uppercase tracking-wider font-semibold text-[#4F6170] dark:text-[#ADBFCE] flex items-center justify-between">
+                                  <span>Upstream Columns</span>
+                                  <span>
+                                    {
+                                      upstreamColumns.filter((col) =>
+                                        col
+                                          .toLowerCase()
+                                          .includes(varSearchQuery.trim().toLowerCase())
+                                      ).length
+                                    }
+                                  </span>
+                                </div>
+                                {upstreamColumns.filter((col) =>
+                                  col
+                                    .toLowerCase()
+                                    .includes(varSearchQuery.trim().toLowerCase())
+                                ).length === 0 ? (
+                                  <div className="text-[11px] font-mono text-[#4F6170] dark:text-[#ADBFCE] italic px-1">
+                                    {upstreamColumns.length === 0
+                                      ? 'No upstream columns defined before this column'
+                                      : 'No matching columns'}
+                                  </div>
+                                ) : (
+                                  <div className="flex flex-wrap gap-1.5 max-h-36 overflow-y-auto pr-1">
+                                    {upstreamColumns
+                                      .filter((col) =>
+                                        col
+                                          .toLowerCase()
+                                          .includes(varSearchQuery.trim().toLowerCase())
+                                      )
+                                      .map((col) => (
                                         <button
                                           key={col}
                                           type="button"
-                                onClick={() => {
-                                  const currentExpr = field.config?.expression || '';
-                                  handleConfigChange(idx, 'expression', currentExpr + col);
-                                }}
-                                className="px-2 py-0.5 rounded bg-[#F4FAFF] dark:bg-[#161C22] border-[1.5px] border-[#DEE3E8] dark:border-[#4F6170] hover:border-[#000000] dark:hover:border-[#FFFFFF] text-[#000000] dark:text-[#FFFFFF] transition-colors"
+                                          onClick={() => handleInsertVariable(idx, col)}
+                                          className="px-2 py-1 text-xs font-mono font-medium rounded-[5px] bg-[#F4FAFF] dark:bg-[#161C22] border-[1.5px] border-[#DEE3E8] dark:border-[#4F6170] hover:border-[#000000] dark:hover:border-[#FFFFFF] text-[#000000] dark:text-[#FFFFFF] transition-colors hover:bg-[#FFFFFF] dark:hover:bg-[#0E1317] cursor-pointer"
                                         >
                                           {col}
                                         </button>
-                            ))
+                                      ))}
+                                  </div>
+                                )}
+                              </div>
+                            </div>
                           )}
                         </div>
                       </div>
@@ -1194,15 +1329,15 @@ export const TableSchemaEditor: React.FC<TableSchemaEditorProps> = ({
                       )}
 
                       {field.type === 'string' && (
-                        <div className="col-span-2">
+                        <div className="col-span-3">
                           <label className="block text-[#4F6170] dark:text-[#ADBFCE] text-[11px] mb-1 font-medium">
-                            Mask Pattern (# = digit, ? = uppercase, * = alphanumeric)
+                            Mask Pattern (# = digit, ? = uppercase, @ = lowercase, % = hex, * = alphanumeric, \ = escape)
                           </label>
                           <input
                             type="text"
                             value={field.config?.mask || ''}
                             onChange={(e) => handleConfigChange(idx, 'mask', e.target.value)}
-                            placeholder="e.g. USR-####"
+                            placeholder="e.g. USR-####, MAC-\#%%-\#%%, 0x%%%%"
                             className="w-full px-2.5 py-1.5 font-mono text-xs border-[1.5px] border-[#DEE3E8] dark:border-[#4F6170] rounded-[5px] bg-[#FFFFFF] dark:bg-[#0E1317] text-[#000000] dark:text-[#FFFFFF] focus:outline-none focus:border-[#000000] dark:focus:border-[#FFFFFF]"
                           />
                         </div>

frontend/src/types/index.ts

@@ -48,6 +48,7 @@ export interface Table {
   archetype: TableArchetype;
   status: TableStatus;
   format: OutputFormat;
+  use_project_seed?: boolean;
   row_limit?: number | null;
   exclude_if?: string | null;
   fields: Field[];