feat: improved special characters usage on randomized strings

backend/engine/samplers.py

@@ -77,11 +77,21 @@ def sample_weighted_without_replacement(
     return [item for _, item in keys[:chosen_size]]
 
 
+HEX_UPPERCASE = "0123456789ABCDEF"
+
+
 def sample_regex_mask(mask: str, rng: random.Random) -> str:
-    # '#' -> digit, '?' -> uppercase, '@' -> lowercase, '*' -> alphanumeric
+    # '#' -> digit, '?' -> uppercase, '@' -> lowercase, '*' -> alphanumeric, '%' -> hex uppercase
+    # '\' -> escape next character literally
     out = []
+    escape = False
     for char in mask:
-        if char == "#":
+        if escape:
+            out.append(char)
+            escape = False
+        elif char == "\\":
+            escape = True
+        elif char == "#":
             out.append(rng.choice(string.digits))
         elif char == "?":
             out.append(rng.choice(string.ascii_uppercase))
@@ -89,8 +99,12 @@ def sample_regex_mask(mask: str, rng: random.Random) -> str:
             out.append(rng.choice(string.ascii_lowercase))
         elif char == "*":
             out.append(rng.choice(string.ascii_letters + string.digits))
+        elif char == "%":
+            out.append(rng.choice(HEX_UPPERCASE))
         else:
             out.append(char)
+    if escape:
+        out.append("\\")
     return "".join(out)
 
 

backend/tests/test_engine.py

@@ -19,7 +19,7 @@ from backend.engine.sandbox import (
     FormulaSyntaxError,
     extract_variable_names,
 )
-from backend.engine.samplers import generate_uuid7, get_random_sublist
+from backend.engine.samplers import generate_uuid7, get_random_sublist, sample_regex_mask
 from backend.engine.writers import StreamingCsvWriter, StreamingParquetWriter
 
 
@@ -657,3 +657,62 @@ def test_extended_builtin_metadata_variables():
     df_agg = pl.concat(chunks_agg)
     assert len(df_agg) == 6
     assert df_agg["meta_calc"].to_list() == ["1/6", "2/6", "3/6", "4/6", "5/6", "6/6"]
+
+
+def test_sample_regex_mask_hex_and_escaping():
+    import random
+    rng = random.Random(42)
+
+    # 1. Hexadecimal token '%' (0-9, A-F)
+    hex_sample = sample_regex_mask("0x%%%%", rng)
+    assert hex_sample.startswith("0x")
+    assert len(hex_sample) == 6
+    for c in hex_sample[2:]:
+        assert c in "0123456789ABCDEF"
+
+    # 2. Escaped special tokens '\#', '\?', '\@', '\*', '\%', '\H'
+    escaped_sample = sample_regex_mask(r"\#\?\@\*\%\Hello", rng)
+    assert escaped_sample == "#?@*%Hello"
+
+    # 3. Escaped backslash '\\'
+    path_sample = sample_regex_mask(r"C:\\DIR\\\#-####", rng)
+    assert path_sample.startswith(r"C:\DIR\#-")
+    assert len(path_sample) == 13
+    assert path_sample[-4:].isdigit()
+
+    # 4. Trailing backslash
+    trailing_sample = sample_regex_mask("TEST-\\\\", rng)
+    assert trailing_sample == "TEST-\\"
+
+    single_trailing = sample_regex_mask("TEST-\\", rng)
+    assert single_trailing == "TEST-\\"
+
+    # 5. Table generator end-to-end integration test with masked string
+    table = {
+        "name": "hex_table",
+        "archetype": "record",
+        "status": "valid",
+        "fields": [
+            {
+                "name": "mac_addr",
+                "role": "randomized",
+                "type": "string",
+                "config": {"mask": r"MAC-\#%%-\#%%-\#%%"},
+                "active": True,
+            }
+        ],
+    }
+    gen = TableGenerator(table, [], seed=77, chunk_size=5)
+    df = next(gen.generate_record_chunks(5))
+    macs = df["mac_addr"].to_list()
+    assert len(macs) == 5
+    for m in macs:
+        assert m.startswith("MAC-#")
+        parts = m.split("-")
+        assert len(parts) == 4
+        assert parts[0] == "MAC"
+        for p in parts[1:]:
+            assert p.startswith("#")
+            assert len(p) == 3
+            assert all(c in "0123456789ABCDEF" for c in p[1:])
+

frontend/src/components/TableSchemaEditor.tsx

@@ -1,4 +1,4 @@
-import React, { useState } from 'react';
+import React, { useEffect, useRef, useState } from 'react';
 import {
   ArrowLeft,
   ArrowUp,
@@ -14,6 +14,8 @@ import {
   Clipboard,
   Check,
   X,
+  BracketsCurly,
+  MagnifyingGlass,
 } from '@phosphor-icons/react';
 import { updateTable, validateTable, previewTable } from '../api';
 import type { ColumnRole, ColumnType, Field, Table } from '../types';
@@ -27,6 +29,16 @@ interface TableSchemaEditorProps {
   onGenerate: (format: 'csv' | 'parquet') => void;
 }
 
+const BUILTIN_METADATA_VARIABLES = [
+  { name: '_row', desc: '1-based row counter' },
+  { name: '_index', desc: '0-based row counter' },
+  { name: '_count', desc: 'Total target rows' },
+  { name: '_name', desc: 'Current table name' },
+  { name: '_project', desc: 'Project name' },
+  { name: '_timestamp', desc: 'UTC ISO timestamp' },
+  { name: '_seed', desc: 'Project RNG seed' },
+];
+
 export const TableSchemaEditor: React.FC<TableSchemaEditorProps> = ({
   table,
   allProjectTables,
@@ -59,6 +71,38 @@ export const TableSchemaEditor: React.FC<TableSchemaEditorProps> = ({
   const [copied, setCopied] = useState(false);
   const [showConfirmGuard, setShowConfirmGuard] = useState(false);
 
+  // Variable Picker Popover state
+  const [activeVarPickerIdx, setActiveVarPickerIdx] = useState<number | null>(null);
+  const [varSearchQuery, setVarSearchQuery] = useState('');
+  const varPickerRef = useRef<HTMLDivElement>(null);
+
+  useEffect(() => {
+    if (activeVarPickerIdx === null) return;
+    const handleClickOutside = (e: MouseEvent) => {
+      if (varPickerRef.current && !varPickerRef.current.contains(e.target as Node)) {
+        setActiveVarPickerIdx(null);
+      }
+    };
+    const handleKeyDown = (e: KeyboardEvent) => {
+      if (e.key === 'Escape') {
+        setActiveVarPickerIdx(null);
+      }
+    };
+    document.addEventListener('mousedown', handleClickOutside);
+    document.addEventListener('keydown', handleKeyDown);
+    return () => {
+      document.removeEventListener('mousedown', handleClickOutside);
+      document.removeEventListener('keydown', handleKeyDown);
+    };
+  }, [activeVarPickerIdx]);
+
+  const handleInsertVariable = (fieldIdx: number, varName: string) => {
+    const currentExpr = fields[fieldIdx]?.config?.expression || '';
+    handleConfigChange(fieldIdx, 'expression', currentExpr + varName);
+    setActiveVarPickerIdx(null);
+    setVarSearchQuery('');
+  };
+
   const isLocked = table.status === 'valid' && table.referencing_tables.length > 0;
   const isAggregate = table.archetype === 'aggregate';
 
@@ -690,102 +734,159 @@ export const TableSchemaEditor: React.FC<TableSchemaEditorProps> = ({
                         <span className="font-medium text-[#000000] dark:text-[#FFFFFF]">
                           Python Expression (Safe Sandbox with f-strings, math, random):
                         </span>
-                        <div className="flex flex-wrap items-center gap-1.5 text-[11px] font-mono text-[#4F6170] dark:text-[#ADBFCE]">
-                          <span>Built-ins:</span>
+
+                        {/* Unified Variable Picker Button & Popover */}
+                        <div className="relative">
                           <button
                             type="button"
-                            onClick={() => {
-                              const currentExpr = field.config?.expression || '';
-                              handleConfigChange(idx, 'expression', currentExpr + '_row');
+                            disabled={isLocked || !isActive}
+                            onClick={(e) => {
+                              e.stopPropagation();
+                              if (activeVarPickerIdx === idx) {
+                                setActiveVarPickerIdx(null);
+                              } else {
+                                setActiveVarPickerIdx(idx);
+                                setVarSearchQuery('');
+                              }
                             }}
-                            className="px-2 py-0.5 rounded bg-[#EDF3EC] dark:bg-[#1b2b1e] border-[1.5px] border-[#DEE3E8] dark:border-[#4F6170] hover:border-[#000000] dark:hover:border-[#FFFFFF] text-[#346538] dark:text-[#88d48e] transition-colors"
-                            title="1-based row counter (1, 2, 3...)"
+                            className={`flex items-center space-x-1.5 text-xs font-mono font-medium px-2.5 py-1 rounded-[5px] border-[1.5px] transition-colors disabled:opacity-40 cursor-pointer ${
+                              activeVarPickerIdx === idx
+                                ? 'border-[#000000] dark:border-[#FFFFFF] bg-[#EDF3EC] dark:bg-[#1b2b1e] text-[#346538] dark:text-[#88d48e]'
+                                : 'border-[#DEE3E8] dark:border-[#4F6170] bg-[#FFFFFF] dark:bg-[#0E1317] hover:border-[#000000] dark:hover:border-[#FFFFFF] text-[#000000] dark:text-[#FFFFFF]'
+                            }`}
+                            title="Insert built-in variable or upstream column into expression"
                           >
-                            _row
+                            <BracketsCurly size={13} weight="bold" />
+                            <span>{'{ }'} Insert Variable</span>
                           </button>
-                          <button
-                            type="button"
-                            onClick={() => {
-                              const currentExpr = field.config?.expression || '';
-                              handleConfigChange(idx, 'expression', currentExpr + '_index');
-                            }}
-                            className="px-2 py-0.5 rounded bg-[#EDF3EC] dark:bg-[#1b2b1e] border-[1.5px] border-[#DEE3E8] dark:border-[#4F6170] hover:border-[#000000] dark:hover:border-[#FFFFFF] text-[#346538] dark:text-[#88d48e] transition-colors"
-                            title="0-based row counter (0, 1, 2...)"
+
+                          {activeVarPickerIdx === idx && (
+                            <div
+                              ref={varPickerRef}
+                              onClick={(e) => e.stopPropagation()}
+                              className="absolute right-0 top-full mt-1.5 w-80 max-h-[380px] overflow-y-auto z-40 bg-[#FFFFFF] dark:bg-[#0E1317] border-[1.5px] border-[#DEE3E8] dark:border-[#4F6170] rounded-[8px] shadow-xl p-3 space-y-3"
                             >
-                            _index
-                          </button>
-                          <button
-                            type="button"
-                            onClick={() => {
-                              const currentExpr = field.config?.expression || '';
-                              handleConfigChange(idx, 'expression', currentExpr + '_count');
-                            }}
-                            className="px-2 py-0.5 rounded bg-[#EDF3EC] dark:bg-[#1b2b1e] border-[1.5px] border-[#DEE3E8] dark:border-[#4F6170] hover:border-[#000000] dark:hover:border-[#FFFFFF] text-[#346538] dark:text-[#88d48e] transition-colors"
-                            title="Total rows target count (int)"
-                          >
-                            _count
-                          </button>
-                          <button
-                            type="button"
-                            onClick={() => {
-                              const currentExpr = field.config?.expression || '';
-                              handleConfigChange(idx, 'expression', currentExpr + '_name');
-                            }}
-                            className="px-2 py-0.5 rounded bg-[#EDF3EC] dark:bg-[#1b2b1e] border-[1.5px] border-[#DEE3E8] dark:border-[#4F6170] hover:border-[#000000] dark:hover:border-[#FFFFFF] text-[#346538] dark:text-[#88d48e] transition-colors"
-                            title="Current table name (str)"
-                          >
-                            _name
-                          </button>
-                          <button
-                            type="button"
-                            onClick={() => {
-                              const currentExpr = field.config?.expression || '';
-                              handleConfigChange(idx, 'expression', currentExpr + '_project');
-                            }}
-                            className="px-2 py-0.5 rounded bg-[#EDF3EC] dark:bg-[#1b2b1e] border-[1.5px] border-[#DEE3E8] dark:border-[#4F6170] hover:border-[#000000] dark:hover:border-[#FFFFFF] text-[#346538] dark:text-[#88d48e] transition-colors"
-                            title="Project name (str)"
-                          >
-                            _project
-                          </button>
-                          <button
-                            type="button"
-                            onClick={() => {
-                              const currentExpr = field.config?.expression || '';
-                              handleConfigChange(idx, 'expression', currentExpr + '_timestamp');
-                            }}
-                            className="px-2 py-0.5 rounded bg-[#EDF3EC] dark:bg-[#1b2b1e] border-[1.5px] border-[#DEE3E8] dark:border-[#4F6170] hover:border-[#000000] dark:hover:border-[#FFFFFF] text-[#346538] dark:text-[#88d48e] transition-colors"
-                            title="UTC generation timestamp in ISO 8601 (str)"
-                          >
-                            _timestamp
-                          </button>
-                          <button
-                            type="button"
-                            onClick={() => {
-                              const currentExpr = field.config?.expression || '';
-                              handleConfigChange(idx, 'expression', currentExpr + '_seed');
-                            }}
-                            className="px-2 py-0.5 rounded bg-[#EDF3EC] dark:bg-[#1b2b1e] border-[1.5px] border-[#DEE3E8] dark:border-[#4F6170] hover:border-[#000000] dark:hover:border-[#FFFFFF] text-[#346538] dark:text-[#88d48e] transition-colors"
-                            title="Project RNG seed (int)"
-                          >
-                            _seed
-                          </button>
-                          <span className="ml-1">Upstream:</span>
-                          {upstreamColumns.length === 0 ? (
-                            <span className="italic">none</span>
+                              {/* Search Input */}
+                              <div className="relative">
+                                <MagnifyingGlass
+                                  size={13}
+                                  className="absolute left-2.5 top-1/2 -translate-y-1/2 text-[#4F6170] dark:text-[#ADBFCE]"
+                                />
+                                <input
+                                  type="text"
+                                  autoFocus
+                                  value={varSearchQuery}
+                                  onChange={(e) => setVarSearchQuery(e.target.value)}
+                                  placeholder="Filter variables..."
+                                  className="w-full pl-8 pr-2.5 py-1.5 text-xs font-mono border-[1.5px] border-[#DEE3E8] dark:border-[#4F6170] rounded-[5px] bg-[#F4FAFF] dark:bg-[#161C22] text-[#000000] dark:text-[#FFFFFF] focus:outline-none focus:border-[#000000] dark:focus:border-[#FFFFFF]"
+                                />
+                              </div>
+
+                              {/* Fundamental / Built-in Variables Section */}
+                              <div className="space-y-1.5">
+                                <div className="text-[10px] font-mono uppercase tracking-wider font-semibold text-[#4F6170] dark:text-[#ADBFCE] flex items-center justify-between">
+                                  <span>Fundamental Variables</span>
+                                  <span>
+                                    {
+                                      BUILTIN_METADATA_VARIABLES.filter(
+                                        (v) =>
+                                          v.name
+                                            .toLowerCase()
+                                            .includes(varSearchQuery.trim().toLowerCase()) ||
+                                          v.desc
+                                            .toLowerCase()
+                                            .includes(varSearchQuery.trim().toLowerCase())
+                                      ).length
+                                    }
+                                  </span>
+                                </div>
+                                {BUILTIN_METADATA_VARIABLES.filter(
+                                  (v) =>
+                                    v.name
+                                      .toLowerCase()
+                                      .includes(varSearchQuery.trim().toLowerCase()) ||
+                                    v.desc
+                                      .toLowerCase()
+                                      .includes(varSearchQuery.trim().toLowerCase())
+                                ).length === 0 ? (
+                                  <div className="text-[11px] font-mono text-[#4F6170] dark:text-[#ADBFCE] italic px-1">
+                                    No matching variables
+                                  </div>
                                 ) : (
-                            upstreamColumns.map((col) => (
+                                  <div className="grid grid-cols-1 gap-1">
+                                    {BUILTIN_METADATA_VARIABLES.filter(
+                                      (v) =>
+                                        v.name
+                                          .toLowerCase()
+                                          .includes(varSearchQuery.trim().toLowerCase()) ||
+                                        v.desc
+                                          .toLowerCase()
+                                          .includes(varSearchQuery.trim().toLowerCase())
+                                    ).map((v) => (
+                                      <button
+                                        key={v.name}
+                                        type="button"
+                                        onClick={() => handleInsertVariable(idx, v.name)}
+                                        className="flex items-center justify-between text-left px-2 py-1.5 rounded-[5px] bg-[#EDF3EC] dark:bg-[#1b2b1e] border border-[#DEE3E8] dark:border-[#4F6170] hover:border-[#000000] dark:hover:border-[#FFFFFF] transition-colors group cursor-pointer"
+                                      >
+                                        <span className="font-mono text-xs font-bold text-[#346538] dark:text-[#88d48e]">
+                                          {v.name}
+                                        </span>
+                                        <span className="font-mono text-[10px] text-[#4F6170] dark:text-[#ADBFCE] group-hover:text-[#000000] dark:group-hover:text-[#FFFFFF] transition-colors truncate max-w-[155px]">
+                                          {v.desc}
+                                        </span>
+                                      </button>
+                                    ))}
+                                  </div>
+                                )}
+                              </div>
+
+                              {/* Upstream Columns Section */}
+                              <div className="space-y-1.5 border-t-[1.5px] border-[#DEE3E8] dark:border-[#4F6170]/60 pt-2">
+                                <div className="text-[10px] font-mono uppercase tracking-wider font-semibold text-[#4F6170] dark:text-[#ADBFCE] flex items-center justify-between">
+                                  <span>Upstream Columns</span>
+                                  <span>
+                                    {
+                                      upstreamColumns.filter((col) =>
+                                        col
+                                          .toLowerCase()
+                                          .includes(varSearchQuery.trim().toLowerCase())
+                                      ).length
+                                    }
+                                  </span>
+                                </div>
+                                {upstreamColumns.filter((col) =>
+                                  col
+                                    .toLowerCase()
+                                    .includes(varSearchQuery.trim().toLowerCase())
+                                ).length === 0 ? (
+                                  <div className="text-[11px] font-mono text-[#4F6170] dark:text-[#ADBFCE] italic px-1">
+                                    {upstreamColumns.length === 0
+                                      ? 'No upstream columns defined before this column'
+                                      : 'No matching columns'}
+                                  </div>
+                                ) : (
+                                  <div className="flex flex-wrap gap-1.5 max-h-36 overflow-y-auto pr-1">
+                                    {upstreamColumns
+                                      .filter((col) =>
+                                        col
+                                          .toLowerCase()
+                                          .includes(varSearchQuery.trim().toLowerCase())
+                                      )
+                                      .map((col) => (
                                         <button
                                           key={col}
                                           type="button"
-                                onClick={() => {
-                                  const currentExpr = field.config?.expression || '';
-                                  handleConfigChange(idx, 'expression', currentExpr + col);
-                                }}
-                                className="px-2 py-0.5 rounded bg-[#F4FAFF] dark:bg-[#161C22] border-[1.5px] border-[#DEE3E8] dark:border-[#4F6170] hover:border-[#000000] dark:hover:border-[#FFFFFF] text-[#000000] dark:text-[#FFFFFF] transition-colors"
+                                          onClick={() => handleInsertVariable(idx, col)}
+                                          className="px-2 py-1 text-xs font-mono font-medium rounded-[5px] bg-[#F4FAFF] dark:bg-[#161C22] border-[1.5px] border-[#DEE3E8] dark:border-[#4F6170] hover:border-[#000000] dark:hover:border-[#FFFFFF] text-[#000000] dark:text-[#FFFFFF] transition-colors hover:bg-[#FFFFFF] dark:hover:bg-[#0E1317] cursor-pointer"
                                         >
                                           {col}
                                         </button>
-                            ))
+                                      ))}
+                                  </div>
+                                )}
+                              </div>
+                            </div>
                           )}
                         </div>
                       </div>
@@ -1194,15 +1295,15 @@ export const TableSchemaEditor: React.FC<TableSchemaEditorProps> = ({
                       )}
 
                       {field.type === 'string' && (
-                        <div className="col-span-2">
+                        <div className="col-span-3">
                           <label className="block text-[#4F6170] dark:text-[#ADBFCE] text-[11px] mb-1 font-medium">
-                            Mask Pattern (# = digit, ? = uppercase, * = alphanumeric)
+                            Mask Pattern (# = digit, ? = uppercase, @ = lowercase, % = hex, * = alphanumeric, \ = escape)
                           </label>
                           <input
                             type="text"
                             value={field.config?.mask || ''}
                             onChange={(e) => handleConfigChange(idx, 'mask', e.target.value)}
-                            placeholder="e.g. USR-####"
+                            placeholder="e.g. USR-####, MAC-\#%%-\#%%, 0x%%%%"
                             className="w-full px-2.5 py-1.5 font-mono text-xs border-[1.5px] border-[#DEE3E8] dark:border-[#4F6170] rounded-[5px] bg-[#FFFFFF] dark:bg-[#0E1317] text-[#000000] dark:text-[#FFFFFF] focus:outline-none focus:border-[#000000] dark:focus:border-[#FFFFFF]"
                           />
                         </div>