Task ID: task_e_682fd6587844832397620c74d830653a
Configuring language runtimes...
+ rm -- /tmp/knJM31-setup_script.sh
+ source ./codex_setup.sh
++ uv sync --frozen
Using CPython 3.12.3 interpreter at: /usr/bin/python3.12
Creating virtual environment at: .venv
Building scubaduck @ file:///workspace/scubaduck
Downloading ruff (11.0MiB)
Downloading pyright (5.3MiB)
Downloading playwright (43.1MiB)
Downloading duckdb (19.3MiB)
Built scubaduck @ file:///workspace/scubaduck
Downloading ruff
Downloading duckdb
Downloading playwright
Downloading pyright
Prepared 33 packages in 2.58s
Installed 33 packages in 103ms
+ blinker==1.9.0
+ certifi==2025.4.26
+ charset-normalizer==3.4.2
+ click==8.2.0
+ duckdb==1.2.2
+ execnet==2.1.1
+ flask==3.1.1
+ greenlet==3.2.2
+ idna==3.10
+ iniconfig==2.1.0
+ itsdangerous==2.2.0
+ jinja2==3.1.6
+ markupsafe==3.0.2
+ nodeenv==1.9.1
+ packaging==25.0
+ playwright==1.52.0
+ pluggy==1.6.0
+ pyee==13.0.0
+ pyright==1.1.400
+ pytest==8.3.5
+ pytest-base-url==2.1.0
+ pytest-playwright==0.7.0
+ pytest-xdist==3.6.1
+ python-dateutil==2.9.0.post0
+ python-slugify==8.0.4
+ requests==2.32.3
+ ruff==0.11.10
+ scubaduck==0.1.0 (from file:///workspace/scubaduck)
+ six==1.17.0
+ text-unidecode==1.3
+ typing-extensions==4.13.2
+ urllib3==2.4.0
+ werkzeug==3.1.3
++ source .venv/bin/activate
+++ '[' -z '' ']'
+++ '[' -n x ']'
+++ SCRIPT_PATH=.venv/bin/activate
+++ '[' .venv/bin/activate = /tmp/knJM31-setup_script.sh ']'
+++ deactivate nondestructive
+++ unset -f pydoc
+++ '[' -z '' ']'
+++ '[' -z '' ']'
+++ hash -r
+++ '[' -z '' ']'
+++ unset VIRTUAL_ENV
+++ unset VIRTUAL_ENV_PROMPT
+++ '[' '!' nondestructive = nondestructive ']'
+++ VIRTUAL_ENV=/workspace/scubaduck/.venv
+++ '[' linux-gnu = cygwin ']'
+++ '[' linux-gnu = msys ']'
+++ export VIRTUAL_ENV
+++ '[' -z '' ']'
+++ unset SCRIPT_PATH
+++ _OLD_VIRTUAL_PATH=/root/.cargo/bin:/root/.rbenv/shims:/root/.rbenv/bin:/root/.rbenv/shims:/root/.local/share/swiftly/bin:/root/.bun/bin:/root/.nvm/versions/node/v22.16.0/bin:/root/.pyenv/shims:3442PYENV_ROOT/shims:/root/.pyenv/bin:/usr/local/go/bin:/root/go/bin:/root/.rbenv/bin:/root/.rbenv/shims:/root/.bun/bin:/root/.local/bin:/root/.pyenv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+++ PATH=/workspace/scubaduck/.venv/bin:/root/.cargo/bin:/root/.rbenv/shims:/root/.rbenv/bin:/root/.rbenv/shims:/root/.local/share/swiftly/bin:/root/.bun/bin:/root/.nvm/versions/node/v22.16.0/bin:/root/.pyenv/shims:3442PYENV_ROOT/shims:/root/.pyenv/bin:/usr/local/go/bin:/root/go/bin:/root/.rbenv/bin:/root/.rbenv/shims:/root/.bun/bin:/root/.local/bin:/root/.pyenv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+++ export PATH
+++ '[' xscubaduck '!=' x ']'
+++ VIRTUAL_ENV_PROMPT=scubaduck
+++ export VIRTUAL_ENV_PROMPT
+++ '[' -z '' ']'
+++ '[' -z '' ']'
+++ _OLD_VIRTUAL_PS1=
+++ PS1='(scubaduck) '
+++ export PS1
+++ alias pydoc
+++ true
+++ hash -r
++ python -c 'import os; import duckdb; con = duckdb.connect(); con.execute(f"SET http_proxy = '\''{os.getenv("HTTP_PROXY")}'\''"); con.execute("INSTALL '\''sqlite'\'';")'
++ playwright install chromium
Downloading Chromium 136.0.7103.25 (playwright build v1169) from https://cdn.playwright.dev/dbazure/download/playwright/builds/chromium/1169/chromium-linux.zip
167.7 MiB [] 0% 18.2s167.7 MiB [] 0% 38.5s167.7 MiB [] 0% 31.4s167.7 MiB [] 0% 15.9s167.7 MiB [] 0% 12.7s167.7 MiB [] 0% 12.2s167.7 MiB [] 1% 11.5s167.7 MiB [] 1% 11.3s167.7 MiB [] 1% 8.9s167.7 MiB [] 1% 8.7s167.7 MiB [] 2% 8.5s167.7 MiB [] 2% 7.9s167.7 MiB [] 3% 6.4s167.7 MiB [] 3% 5.7s167.7 MiB [] 4% 4.9s167.7 MiB [] 5% 4.7s167.7 MiB [] 5% 4.5s167.7 MiB [] 6% 4.0s167.7 MiB [] 7% 3.7s167.7 MiB [] 9% 3.3s167.7 MiB [] 9% 3.2s167.7 MiB [] 10% 3.1s167.7 MiB [] 12% 2.6s167.7 MiB [] 14% 2.4s167.7 MiB [] 15% 2.3s167.7 MiB [] 16% 2.3s167.7 MiB [] 17% 2.2s167.7 MiB [] 18% 2.1s167.7 MiB [] 19% 2.0s167.7 MiB [] 20% 1.9s167.7 MiB [] 21% 1.9s167.7 MiB [] 22% 1.8s167.7 MiB [] 24% 1.7s167.7 MiB [] 26% 1.6s167.7 MiB [] 27% 1.6s167.7 MiB [] 29% 1.5s167.7 MiB [] 30% 1.5s167.7 MiB [] 31% 1.4s167.7 MiB [] 33% 1.3s167.7 MiB [] 34% 1.3s167.7 MiB [] 35% 1.3s167.7 MiB [] 37% 1.2s167.7 MiB [] 39% 1.1s167.7 MiB [] 40% 1.1s167.7 MiB [] 41% 1.1s167.7 MiB [] 42% 1.1s167.7 MiB [] 43% 1.0s167.7 MiB [] 45% 1.0s167.7 MiB [] 47% 0.9s167.7 MiB [] 48% 0.9s167.7 MiB [] 50% 0.9s167.7 MiB [] 51% 0.8s167.7 MiB [] 52% 0.8s167.7 MiB [] 54% 0.8s167.7 MiB [] 55% 0.7s167.7 MiB [] 56% 0.7s167.7 MiB [] 59% 0.7s167.7 MiB [] 61% 0.6s167.7 MiB [] 63% 0.6s167.7 MiB [] 64% 0.6s167.7 MiB [] 65% 0.6s167.7 MiB [] 66% 0.5s167.7 MiB [] 68% 0.5s167.7 MiB [] 69% 0.5s167.7 MiB [] 71% 0.4s167.7 MiB [] 72% 0.4s167.7 MiB [] 74% 0.4s167.7 MiB [] 76% 0.4s167.7 MiB [] 77% 0.3s167.7 MiB [] 79% 0.3s167.7 MiB [] 81% 0.3s167.7 MiB [] 83% 0.2s167.7 MiB [] 84% 0.2s167.7 MiB [] 86% 0.2s167.7 MiB [] 87% 0.2s167.7 MiB [] 89% 0.2s167.7 MiB [] 90% 0.1s167.7 MiB [] 92% 0.1s167.7 MiB [] 94% 0.1s167.7 MiB [] 95% 0.1s167.7 MiB [] 97% 0.0s167.7 MiB [] 99% 0.0s167.7 MiB [] 100% 0.0s
Chromium 136.0.7103.25 (playwright build v1169) downloaded to /root/.cache/ms-playwright/chromium-1169
Downloading FFMPEG playwright build v1011 from https://cdn.playwright.dev/dbazure/download/playwright/builds/ffmpeg/1011/ffmpeg-linux.zip
2.3 MiB [] 0% 0.0s2.3 MiB [] 4% 0.5s2.3 MiB [] 11% 0.3s2.3 MiB [] 22% 0.2s2.3 MiB [] 52% 0.1s2.3 MiB [] 99% 0.0s2.3 MiB [] 100% 0.0s
FFMPEG playwright build v1011 downloaded to /root/.cache/ms-playwright/ffmpeg-1011
Downloading Chromium Headless Shell 136.0.7103.25 (playwright build v1169) from https://cdn.playwright.dev/dbazure/download/playwright/builds/chromium/1169/chromium-headless-shell-linux.zip
101.4 MiB [] 0% 0.0s101.4 MiB [] 0% 17.9s101.4 MiB [] 0% 18.1s101.4 MiB [] 0% 17.1s101.4 MiB [] 0% 19.3s101.4 MiB [] 0% 15.8s101.4 MiB [] 1% 8.3s101.4 MiB [] 2% 7.2s101.4 MiB [] 2% 6.0s101.4 MiB [] 3% 6.1s101.4 MiB [] 3% 5.5s101.4 MiB [] 5% 4.4s101.4 MiB [] 6% 3.9s101.4 MiB [] 8% 3.2s101.4 MiB [] 9% 3.0s101.4 MiB [] 10% 2.8s101.4 MiB [] 12% 2.4s101.4 MiB [] 14% 2.2s101.4 MiB [] 15% 2.2s101.4 MiB [] 18% 1.8s101.4 MiB [] 20% 1.7s101.4 MiB [] 22% 1.5s101.4 MiB [] 25% 1.4s101.4 MiB [] 27% 1.3s101.4 MiB [] 28% 1.3s101.4 MiB [] 30% 1.2s101.4 MiB [] 32% 1.1s101.4 MiB [] 35% 1.0s101.4 MiB [] 36% 1.0s101.4 MiB [] 38% 0.9s101.4 MiB [] 41% 0.8s101.4 MiB [] 43% 0.8s101.4 MiB [] 45% 0.7s101.4 MiB [] 47% 0.7s101.4 MiB [] 51% 0.6s101.4 MiB [] 55% 0.5s101.4 MiB [] 57% 0.5s101.4 MiB [] 60% 0.5s101.4 MiB [] 63% 0.4s101.4 MiB [] 67% 0.4s101.4 MiB [] 70% 0.3s101.4 MiB [] 73% 0.3s101.4 MiB [] 77% 0.2s101.4 MiB [] 78% 0.2s101.4 MiB [] 79% 0.2s101.4 MiB [] 81% 0.2s101.4 MiB [] 85% 0.1s101.4 MiB [] 88% 0.1s101.4 MiB [] 89% 0.1s101.4 MiB [] 92% 0.1s101.4 MiB [] 96% 0.0s101.4 MiB [] 97% 0.0s101.4 MiB [] 100% 0.0s
Chromium Headless Shell 136.0.7103.25 (playwright build v1169) downloaded to /root/.cache/ms-playwright/chromium_headless_shell-1169
++ echo 'source .venv/bin/activate'
I'll consult AGENTS.md for any guidance before proceeding with the changes.
bash: .venv/bin/activate: No such file or directory
root@b8a745a412a8:/workspace# ls
scubaduck
root@b8a745a412a8:/workspace# cd scubaduck
root@b8a745a412a8:/workspace/scubaduck# ls
AGENTS.md codex_setup.sh scubaduck tests
README.md pyproject.toml stubs uv.lock
root@b8a745a412a8:/workspace/scubaduck# cat AGENTS.md
- Use "source .venv/bin/activate" to activate the virtual environment before
running any commands
- ONLY use non-interactive commands like cat, sed, apply_patch to do edits.
Do NOT use interactive editors.
- Do NOT attempt to install packages. Only the packages specified in
pyproject.toml are available. You cannot add new packages. If you
desperately want another package, make a note of it in the final PR
description.
- Use conventional commits to format PR title
- There are no nested AGENTS.md files, this is the only agents file
- When using Playwright, ONLY use chromium browser. This browser is already
installed.
- Use "ruff check" to check lint, "ruff format" to autoformat files and
"pyright" to typecheck.
- When writing the PR description, include the original prompt supplied by the
user.
- When you add functionality to the server, add server tests to
tests/test_server.py. When you add features to the frontend, add frontend
tests to tests/test_web.py
I'll activate .venv, use ruff, pyright, add tests, and ensure chromium for Playwright. Time to check the README.
root@b8a745a412a8:/workspace/scubaduck# ls
AGENTS.md codex_setup.sh scubaduck tests
README.md pyproject.toml stubs uv.lock
root@b8a745a412a8:/workspace/scubaduck# cat README.md | head
I really like Scuba (Meta's internal real-time database system). The distributed
, real-time database part of Scuba is quite difficult (and expensive) to replica
te, but I also really like Scuba's UI for doing queries, and I have found myself
wishing that I have access to it even for "small" databases, e.g., I have a sql
ite dataset I want to explore.
Pivotal ideas:
* Time series by default. In the dedicated "time series" view, there are many fe
atures specifically oriented towards working towards tables that represent event
s that occurred over time: the start, end, compare, aggregate and granularity fi
elds all specially privilege the timestamp field. In fact, you can't log events
to Scuba's backing data store without a timestamp, they always come with one. (S
cuba also supports other views that don't presuppose a time series, but the time
series is the most beloved and well used view.) This is in contrast to typical
software which tries to generalize to arbitrary data first, with time series bei
ng added on later.
* It's all about exploration. Scuba is predicated on the idea that you don't kno
w what you're looking for, that you are going to spend time tweaking queries and
changing filters/grouping as part of an investigation to figure out why a syste
m behaves the way it is. So the filters/comparisons/groupings you want to edit a
re always visible on the left sidebar, with the expectation that you're going to
tweak the query to look at something else. Similarly, all the parameters of you
r query get saved into your URL, so your browser history can double up as a quer
y history / you can easily share a query with someone else. This is contrast to
typical software which is often oriented to making pretty dashboards and reports
. (This function is important too, but it's not what I want in exploration mode!
)
* You can fix data problems in the query editor. It's pretty common to have mess
ed up and ended up with a database that doesn't have exactly the columns you nee
d, or some columns that are corrupted in some way. Scuba has pretty robust suppo
rt for defining custom columns with arbitrary SQL functions, grouping over them
as if they were native functions, and doing so with minimal runtime cost (Scuba
aims to turn around your query in milliseconds!) Having to go and run a huge dat
a pipeline to fix your data is a big impediment to exploration; quick and easy c
ustom columns means you can patch over problems when you're investigating and fi
x them for real later.
I’ll look at the TEST dataset and check the tests related to displaying column headings.
root@b8a745a412a8:/workspace/scubaduck# ls tests
conftest.py test_server_time.py test_web_graph.py
test_multi_table_web.py test_server_timeseries.py test_web_table.py
test_server.py test_web.py test_web_timeseries.py
test_server_basic.py test_web_basics.py web_utils.py
test_server_db_types.py test_web_chips.py
test_server_errors.py test_web_columns.py
root@b8a745a412a8:/workspace/scubaduck# ls scubaduck
__init__.py sample.csv server.py static
I'll check the server and tests to gather more details.
root@b8a745a412a8:/workspace/scubaduck# cat scubaduck/server.py | head -n 40
from __future__ import annotations
from dataclasses import dataclass, field, replace
from typing import Any, Dict, List, Tuple, cast
import re
from datetime import datetime, timedelta, timezone
import time
from pathlib import Path
import os
import traceback
import math
import duckdb
from dateutil import parser as dtparser
from dateutil.relativedelta import relativedelta
from flask import Flask, jsonify, request, send_from_directory
def _quote(ident: str) -> str:
"""Return identifier quoted for SQL."""
return f'"{ident.replace('"', '""')}"'
@dataclass
class Filter:
column: str
op: str
value: str | int | float | list[str] | None
@dataclass
class QueryParams:
start: str | None = None
end: str | None = None
order_by: str | None = None
order_dir: str = "ASC"
limit: int | None = None
columns: list[str] = field(default_factory=lambda: [])
I'm moving straight to the table view logic now.
root@b8a745a412a8:/workspace/scubaduck# grep -n "table" -n scubaduck/server.py |
head
50: table: str = "events"
69: tables = [
72: "SELECT name FROM sqlite_master WHERE type='table'"
75: for t in tables:
83: """Return a DuckDB connection with a small multi-table dataset."""
346: lines = [f"SELECT {select_clause}", f'FROM "{params.table}"']
408: tables = [r[0] for r in con.execute("SHOW TABLES").fetchall()]
409: if not tables:
410: raise ValueError("No tables found in database")
411: default_table = tables[0]
I'm looking for "table view" or "aggregate" in the server files.
root@b8a745a412a8:/workspace/scubaduck# grep -n "aggregate" -n scubaduck/server.
py | head
45: aggregate: str | None = None
271: has_agg = bool(group_cols) or params.aggregate is not None
277: agg = (params.aggregate or "count").lower()
507: aggregate=payload.get("aggregate"),
531: params.group_by or params.aggregate or params.show_hits
536: "error": "group_by, aggregate and show_hits are only
valid for table or timeseries view"
575: agg = (params.aggregate or "count").lower()
root@b8a745a412a8:/workspace/scubaduck# sed -n '250,320p' scubaduck/server.py
group_cols = params.group_by[:]
selected_for_order = set(params.columns) | set(params.derived_columns.keys()
)
if params.graph_type == "timeseries":
sec = _granularity_seconds(params.granularity, params.start, params.end)
x_axis = params.x_axis or params.time_column
if x_axis is None:
raise ValueError("x_axis required for timeseries")
xexpr = _time_expr(x_axis, column_types, params.time_unit)
if params.start:
bucket_expr = (
f"TIMESTAMP '{params.start}' + INTERVAL '{sec} second' * "
f"CAST(floor((epoch({xexpr}) - epoch(TIMESTAMP '{params.start}')
)/{sec}) AS BIGINT)"
)
else:
bucket_expr = (
f"TIMESTAMP 'epoch' + INTERVAL '{sec} second' * "
f"CAST(floor(epoch({xexpr})/{sec}) AS BIGINT)"
)
select_parts.append(f"{bucket_expr} AS bucket")
group_cols = ["bucket"] + group_cols
selected_for_order.add("bucket")
has_agg = bool(group_cols) or params.aggregate is not None
if has_agg:
select_cols = (
group_cols[1:] if params.graph_type == "timeseries" else group_cols
)
select_parts.extend(_quote(c) for c in select_cols)
agg = (params.aggregate or "count").lower()
selected_for_order.update(group_cols)
def agg_expr(col: str) -> str:
expr = _quote(col)
ctype = column_types.get(col, "").upper() if column_types else ""
if "BOOL" in ctype:
expr = f"CAST({_quote(col)} AS BIGINT)"
if agg.startswith("p"):
quant = float(agg[1:]) / 100
return f"quantile({expr}, {quant})"
if agg == "count distinct":
return f"count(DISTINCT {expr})"
if agg == "avg" and column_types is not None:
if "TIMESTAMP" in ctype or "DATE" in ctype or "TIME" in ctype:
return (
"TIMESTAMP 'epoch' + INTERVAL '1 second' * "
f"CAST(avg(epoch({_quote(col)})) AS BIGINT)"
)
return f"{agg}({expr})"
if agg == "count":
select_parts.append("count(*) AS Count")
selected_for_order.add("Count")
else:
for col in params.columns:
if col in group_cols:
continue
select_parts.append(f"{agg_expr(col)} AS {_quote(col)}")
selected_for_order.add(col)
if params.show_hits:
select_parts.insert(len(group_cols), "count(*) AS Hits")
selected_for_order.add("Hits")
else:
select_parts.extend(_quote(c) for c in params.columns)
selected_for_order.update(params.columns)
order_by = params.order_by if params.order_by in selected_for_order else Non
e
if has_agg and params.derived_columns:
inner_params = replace(
params,
derived_columns={},
order_by=None,
I'm moving through more lines to find what I need.
root@b8a745a412a8:/workspace/scubaduck# sed -n '320,420p' scubaduck/server.py
order_by=None,
limit=None,
)
inner_sql = build_query(inner_params, column_types)
outer_select = ["t.*"] + [
f"{expr} AS {name}" for name, expr in params.derived_columns.items()
]
indented_inner = "\n".join(" " + line for line in inner_sql.splitline
s())
lines = [
f"SELECT {', '.join(outer_select)}",
"FROM (",
indented_inner,
") t",
]
if order_by:
lines.append(f"ORDER BY {_quote(order_by)} {params.order_dir}")
elif params.graph_type == "timeseries":
lines.append("ORDER BY bucket")
if params.limit is not None:
lines.append(f"LIMIT {params.limit}")
return "\n".join(lines)
for name, expr in params.derived_columns.items():
select_parts.append(f"{expr} AS {name}")
selected_for_order.add(name)
select_clause = ", ".join(select_parts) if select_parts else "*"
lines = [f"SELECT {select_clause}", f'FROM "{params.table}"']
where_parts: list[str] = []
if params.time_column:
time_expr = _time_expr(params.time_column, column_types, params.time_uni
t)
else:
time_expr = None
if time_expr and params.start:
where_parts.append(f"{time_expr} >= '{params.start}'")
if time_expr and params.end:
where_parts.append(f"{time_expr} <= '{params.end}'")
for f in params.filters:
op = f.op
if op in {"empty", "!empty"}:
val = "''"
else:
if f.value is None:
continue
if isinstance(f.value, list):
if not f.value:
continue
if op == "=":
qcol = _quote(f.column)
vals = " OR ".join(f"{qcol} = '{v}'" for v in f.value)
where_parts.append(f"({vals})")
continue
val = f"'{f.value}'" if isinstance(f.value, str) else str(f.value)
qcol = _quote(f.column)
if op == "contains":
where_parts.append(f"{qcol} ILIKE '%' || {val} || '%'")
elif op == "!contains":
where_parts.append(f"{qcol} NOT ILIKE '%' || {val} || '%'")
elif op == "empty":
where_parts.append(f"{qcol} = {val}")
elif op == "!empty":
where_parts.append(f"{qcol} != {val}")
else:
where_parts.append(f"{qcol} {op} {val}")
if where_parts:
lines.append("WHERE " + " AND ".join(where_parts))
if group_cols:
lines.append("GROUP BY " + ", ".join(_quote(c) for c in group_cols))
if order_by:
lines.append(f"ORDER BY {_quote(order_by)} {params.order_dir}")
elif params.graph_type == "timeseries":
lines.append("ORDER BY bucket")
if params.limit is not None:
lines.append(f"LIMIT {params.limit}")
return "\n".join(lines)
def create_app(db_file: str | Path | None = None) -> Flask:
app = Flask(__name__, static_folder="static")
if db_file is None:
env_db = os.environ.get("SCUBADUCK_DB")
if env_db:
db_file = env_db
if isinstance(db_file, str) and db_file.upper() == "TEST":
con = _create_test_database()
else:
db_path = Path(db_file or Path(__file__).with_name("sample.csv")).resolv
e()
con = _load_database(db_path)
tables = [r[0] for r in con.execute("SHOW TABLES").fetchall()]
if not tables:
raise ValueError("No tables found in database")
default_table = tables[0]
columns_cache: Dict[str, Dict[str, str]] = {}
def get_columns(table: str) -> Dict[str, str]:
if table not in columns_cache:
rows = con.execute(f'PRAGMA table_info("{table}")').fetchall()
if not rows:
raise ValueError(f"Unknown table: {table}")
columns_cache[table] = {r[1]: r[2] for r in rows}
return columns_cache[table]
I'm opening tests/test_web_table.py to check how table views are tested.
root@b8a745a412a8:/workspace/scubaduck# sed -n '1,160p' tests/test_web_table.py
from __future__ import annotations
from typing import Any
from tests.web_utils import run_query
def test_table_sorting(page: Any, server_url: str) -> None:
run_query(
page,
server_url,
start="2024-01-01 00:00:00",
end="2024-01-03 00:00:00",
order_by="user",
order_dir="ASC",
limit=100,
)
# header alignment
align = page.evaluate(
"getComputedStyle(document.querySelector('#results th')).textAlign"
)
assert align == "left"
header = page.locator("#results th").nth(3)
def values() -> list[str]:
return page.locator("#results td:nth-child(4)").all_inner_texts()
orig_rows = values()
assert orig_rows == ["alice", "bob", "alice", "charlie"]
first_sql = page.evaluate("window.lastResults.sql")
header.click()
assert values() == sorted(orig_rows)
assert header.inner_text().endswith("▲")
color = page.evaluate(
"getComputedStyle(document.querySelector('#results th:nth-child(4)')).co
lor"
)
assert "0, 0, 255" in color
assert page.evaluate("window.lastResults.sql") == first_sql
header.click()
assert values() == sorted(orig_rows, reverse=True)
assert header.inner_text().endswith("▼")
header.click()
assert values() == orig_rows
assert header.inner_text() == "user"
color = page.evaluate(
"getComputedStyle(document.querySelector('#results th:nth-child(4)')).co
lor"
)
assert "0, 0, 255" not in color
def test_table_avg_group_by(page: Any, server_url: str) -> None:
data = run_query(
page,
server_url,
start="2024-01-01 00:00:00",
end="2024-01-03 00:00:00",
order_by="timestamp",
group_by=["user"],
aggregate="Avg",
)
assert "error" not in data
assert len(data["rows"]) == 3
def test_table_enhancements(page: Any, server_url: str) -> None:
run_query(
page,
server_url,
start="2024-01-01 00:00:00",
end="2024-01-03 00:00:00",
order_by="timestamp",
limit=10,
)
border = page.evaluate(
"getComputedStyle(document.querySelector('#results td')).borderStyle"
)
assert border == "solid"
color1 = page.evaluate(
"getComputedStyle(document.querySelector('#results tr:nth-child(2) td'))
.backgroundColor"
)
color2 = page.evaluate(
"getComputedStyle(document.querySelector('#results tr:nth-child(3) td'))
.backgroundColor"
)
assert color1 != color2
page.hover("#results tr:nth-child(2)")
hover_color = page.evaluate(
"getComputedStyle(document.querySelector('#results tr:nth-child(2) td'))
.backgroundColor"
)
assert hover_color != color1
page.click("#results tr:nth-child(2)")
selected_color = page.evaluate(
"getComputedStyle(document.querySelector('#results tr:nth-child(2) td'))
.backgroundColor"
)
assert "189, 228, 255" in selected_color
overflow = page.evaluate(
"var v=document.getElementById('view'); v.scrollWidth > v.clientWidth"
)
assert not overflow
def test_table_single_selection(page: Any, server_url: str) -> None:
run_query(
page,
server_url,
start="2024-01-01 00:00:00",
end="2024-01-03 00:00:00",
order_by="timestamp",
limit=10,
)
page.click("#results tr:nth-child(2)")
page.click("#results tr:nth-child(3)")
count = page.evaluate("document.querySelectorAll('#results tr.selected').len
gth")
assert count == 1
is_third = page.evaluate(
"document.querySelector('#results tr:nth-child(3)').classList.contains('
selected')"
)
assert is_third
def test_timestamp_rendering(page: Any, server_url: str) -> None:
run_query(
page,
server_url,
start="2024-01-01 00:00:00",
end="2024-01-02 00:00:00",
order_by="timestamp",
limit=1,
)
cell = page.text_content("#results td")
assert cell != "Invalid Date"
valid = page.evaluate("v => !isNaN(Date.parse(v))", cell)
assert valid
def test_empty_data_message(page: Any, server_url: str) -> None:
data = run_query(
page,
server_url,
start="2025-01-01 00:00:00",
end="2025-01-02 00:00:00",
order_by="timestamp",
limit=100,
)
assert data["rows"] == []
msg = page.text_content("#view")
assert "Empty data provided to table" in msg
def test_group_by_chip_from_url(page: Any, server_url: str) -> None:
url = f"{server_url}?graph_type=table&group_by=user&order_by=user&limit=10"
page.goto(url)
root@b8a745a412a8:/workspace/scubaduck# sed -n '160,320p' tests/test_web_table.p
y
page.goto(url)
page.wait_for_selector("#group_by_field .chip", state="attached")
chips = page.evaluate(
"Array.from(document.querySelectorAll('#group_by_field .chip')).map(c =>
c.firstChild.textContent)"
)
assert chips == ["user"]
def test_group_by_autocomplete(page: Any, server_url: str) -> None:
page.goto(f"{server_url}?graph_type=table")
page.wait_for_selector("#group_by_field", state="visible")
inp = page.query_selector("#group_by_field .f-val")
assert inp
inp.click()
page.keyboard.type("us")
page.wait_for_selector("#group_by_field .chip-dropdown div")
options = page.locator("#group_by_field .chip-dropdown div").all_inner_texts
()
assert "user" in options
def test_group_by_copy_icon(page: Any, server_url: str) -> None:
page.goto(f"{server_url}?graph_type=table")
page.wait_for_selector("#group_by_field", state="visible")
icon = page.text_content("#group_by_field .chip-copy")
assert icon == "⎘"
def test_group_by_input_no_border(page: Any, server_url: str) -> None:
page.goto(f"{server_url}?graph_type=table")
page.wait_for_selector("#group_by_field", state="visible")
border = page.evaluate(
"getComputedStyle(document.querySelector('#group_by_field .f-val')).bord
erStyle"
)
assert border == "none"
def test_table_group_by_query(page: Any, server_url: str) -> None:
data = run_query(
page,
server_url,
start="2024-01-01 00:00:00",
end="2024-01-03 00:00:00",
order_by="user",
limit=100,
group_by=["user"],
aggregate="Count",
)
assert "error" not in data
assert len(data["rows"]) == 3
def test_table_avg_no_group_by(page: Any, server_url: str) -> None:
data = run_query(
page,
server_url,
aggregate="Avg",
)
assert len(data["rows"]) == 1
row = data["rows"][0]
assert row[0] == 4
from dateutil import parser
ts = parser.parse(row[1]).replace(tzinfo=None)
assert ts == parser.parse("2024-01-01 13:00:00")
assert row[2] == 25
def test_table_headers_show_aggregate(page: Any, server_url: str) -> None:
run_query(
page,
server_url,
aggregate="Avg",
)
headers = page.locator("#results th").all_inner_texts()
assert "Hits" in headers
assert "timestamp (avg)" in headers
assert "value (avg)" in headers
def test_format_number_function(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
vals = page.evaluate(
"() => [formatNumber(815210), formatNumber(999.999), formatNumber(0.0004
), formatNumber(0)]"
)
assert vals == ["815.21 K", "999.999", "0.000", "0"]
def test_numeric_cell_nowrap(page: Any, server_url: str) -> None:
run_query(page, server_url, limit=10)
whitespace = page.evaluate(
"getComputedStyle(document.querySelector('#results td:nth-child(3)')).wh
iteSpace"
)
assert whitespace == "nowrap"
def test_date_cell_nowrap(page: Any, server_url: str) -> None:
run_query(page, server_url, limit=10)
whitespace = page.evaluate(
"getComputedStyle(document.querySelector('#results td:nth-child(1)')).wh
iteSpace"
)
assert whitespace == "nowrap"
def test_derived_column_query(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
page.click("text=Columns")
page.click("text=Add Derived")
expr = page.query_selector("#derived_list .derived textarea")
assert expr
expr.fill("value * 2")
page.click("text=View Settings")
page.fill("#start", "2024-01-01 00:00:00")
page.fill("#end", "2024-01-03 00:00:00")
page.fill("#limit", "10")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
data = page.evaluate("window.lastResults")
assert data["rows"][0][-1] == 20
def test_derived_column_remove(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
page.click("text=Columns")
page.click("text=Add Derived")
assert page.query_selector("#derived_list .derived button.remove")
page.click("#derived_list .derived button.remove")
count = page.evaluate("document.querySelectorAll('#derived_list .derived').l
ength")
assert count == 0
def test_sql_query_display(page: Any, server_url: str) -> None:
data = run_query(
page,
server_url,
start="2024-01-01 00:00:00",
end="2024-01-02 00:00:00",
order_by="timestamp",
limit=10,
)
sql = data["sql"]
displayed = page.text_content("#sql_query")
assert displayed is not None
assert displayed.strip() == sql
I’m continuing to scroll past 320 lines to find tests tied to table.
root@b8a745a412a8:/workspace/scubaduck# sed -n '320,640p' tests/test_web_table.p
y
I'll take a look at tests/test_web_chips.py to understand the context.
root@b8a745a412a8:/workspace/scubaduck# sed -n '1,80p' tests/test_web_timeseries
.py
from __future__ import annotations
from typing import Any
from tests.web_utils import select_value
def test_timeseries_default_query(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#graph_type", state="attached")
select_value(page, "#graph_type", "timeseries")
page.click("text=Columns")
page.check("#column_groups input[value='value']")
page.click("text=View Settings")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
data = page.evaluate("window.lastResults")
assert "error" not in data
assert page.is_visible("#chart")
page.click("text=Columns")
assert not page.is_checked("#column_groups input[value='timestamp']")
def test_timeseries_single_bucket(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#graph_type", state="attached")
page.fill("#start", "2024-01-01 00:00:00")
page.fill("#end", "2024-01-01 00:00:00")
select_value(page, "#graph_type", "timeseries")
page.click("text=Columns")
page.check("#column_groups input[value='value']")
page.click("text=View Settings")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
path = page.get_attribute("#chart path", "d")
assert path is not None and "NaN" not in path
def test_timeseries_fill_options(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#graph_type", state="attached")
page.fill("#start", "2024-01-01 00:00:00")
page.fill("#end", "2024-01-02 03:00:00")
select_value(page, "#graph_type", "timeseries")
page.click("text=Columns")
page.check("#column_groups input[value='value']")
page.click("text=View Settings")
select_value(page, "#granularity", "1 hour")
select_value(page, "#fill", "0")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
path_zero = page.get_attribute("#chart path", "d")
assert path_zero is not None and path_zero.count("L") > 20
select_value(page, "#fill", "connect")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
path_conn = page.get_attribute("#chart path", "d")
assert path_conn is not None and path_conn.count("M") == 1
select_value(page, "#fill", "blank")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
path_blank = page.get_attribute("#chart path", "d")
assert path_blank is not None and path_blank.count("M") > 1
def test_timeseries_hover_highlight(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#graph_type", state="attached")
select_value(page, "#graph_type", "timeseries")
page.click("text=Columns")
page.check("#column_groups input[value='value']")
page.click("text=View Settings")
I think the table view logic is in static JS; I'll check "scubaduck/static".
root@b8a745a412a8:/workspace/scubaduck# ls scubaduck/static
index.html js
root@b8a745a412a8:/workspace/scubaduck# ls scubaduck/static/js
chip_input.js table_view.js timeseries_chart.js
sidebar.js time_column.js view_settings.js
Let's open the table_view.js to explore what’s inside.
root@b8a745a412a8:/workspace/scubaduck# sed -n '1,160p' scubaduck/static/js/tabl
e_view.js
// Logic for rendering the table based views. Extracted from index.html so that
// the inline script only handles wiring up the UI.
let originalRows = [];
let sortState = { index: null, dir: null };
function renderTable(rows) {
const table = document.getElementById("results");
table.innerHTML = "";
if (rows.length === 0) return;
let hitsIndex = selectedColumns.indexOf("Hits");
let totalHits = 0;
if (hitsIndex !== -1) {
totalHits = rows.reduce((s, r) => s + Number(r[hitsIndex]), 0);
}
const header = document.createElement("tr");
selectedColumns.forEach((col, i) => {
const th = document.createElement("th");
let label = col;
if (
displayType === "table" &&
col !== "Hits" &&
!(groupBy.chips || []).includes(col)
) {
const agg = document.getElementById("aggregate").value.toLowerCase();
label += ` (${agg})`;
}
th.textContent = label;
th.dataset.index = i;
th.addEventListener("click", handleSort);
if (sortState.index === i) {
th.classList.add("sorted");
th.textContent = label + (sortState.dir === "desc" ? " \u25BC" : " \u25B2"
);
}
th.style.textAlign = "left";
header.appendChild(th);
});
table.appendChild(header);
rows.forEach((row) => {
const tr = document.createElement("tr");
tr.addEventListener("click", () => {
const wasSelected = tr.classList.contains("selected");
document
.querySelectorAll("#results tr.selected")
.forEach((el) => el.classList.remove("selected"));
if (!wasSelected) {
tr.classList.add("selected");
}
});
row.forEach((v, i) => {
const col = selectedColumns[i];
const td = document.createElement("td");
if (isTimeColumn(col)) {
let d;
const t = (columnTypes[col] || "").toUpperCase();
if (t.includes("TIMESTAMP") || t.includes("DATE") || t.includes("TIME"))
{
d = new Date(v);
} else {
const unit = document.getElementById("time_unit").value;
const factors = { s: 1000, ms: 1, us: 0.001, ns: 0.000001 };
d = new Date(Number(v) * (factors[unit] || 1000));
}
td.textContent = d.toLocaleString("en-US", {
weekday: "short",
month: "short",
day: "numeric",
year: "numeric",
hour: "numeric",
minute: "numeric",
second: "numeric",
hour12: true,
timeZoneName: "short",
});
td.classList.add("date");
} else {
if (col === "Hits") {
const pct = totalHits ? ((v / totalHits) * 100).toFixed(1) : "0";
td.textContent = `${formatNumber(v)} (${pct}%)`;
} else {
td.textContent = isStringColumn(col) ? v : formatNumber(v);
}
}
if (!isStringColumn(col) && !isTimeColumn(col)) {
td.classList.add("numeric");
}
td.style.textAlign = isStringColumn(col) ? "left" : "right";
tr.appendChild(td);
});
table.appendChild(tr);
});
// ensure table does not overflow unless necessary
const view = document.getElementById("view");
if (table.scrollWidth <= view.clientWidth) {
table.style.width = "100%";
}
}
function handleSort(e) {
const idx = parseInt(e.target.dataset.index, 10);
if (sortState.index !== idx) {
sortState.index = idx;
sortState.dir = "asc";
} else if (sortState.dir === "asc") {
sortState.dir = "desc";
} else if (sortState.dir === "desc") {
sortState.index = null;
sortState.dir = null;
} else {
sortState.dir = "asc";
}
let rows = originalRows.slice();
if (sortState.index !== null) {
rows.sort((a, b) => {
const va = a[sortState.index];
const vb = b[sortState.index];
if (va === vb) return 0;
if (sortState.dir === "desc") return va < vb ? 1 : -1;
return va > vb ? 1 : -1;
});
}
renderTable(rows);
}
function showResults(data) {
window.lastResults = data;
const view = document.getElementById("view");
if (graphTypeSel.value === "timeseries") {
showTimeSeries(data);
} else {
if (data.rows.length === 0) {
view.innerHTML =
'<p id="empty-message">Empty data provided to table</p><table id="result
s"></table>';
} else {
view.innerHTML = '<table id="results"></table>';
}
originalRows = data.rows.slice();
sortState = { index: null, dir: null };
renderTable(originalRows);
}
const sqlEl = document.createElement("pre");
sqlEl.id = "sql_query";
sqlEl.style.whiteSpace = "pre-wrap";
sqlEl.style.marginTop = "10px";
sqlEl.textContent = data.sql;
view.appendChild(sqlEl);
document.getElementById("query_info").textContent = `Your query took about ${l
astQueryTime} ms`;
}
function showError(err) {
window.lastResults = err;
const view = document.getElementById("view");
let msg = "";
if (typeof err === "string") {
msg = err;
} else if (err) {
msg = err.error || "Error";
if (err.sql) {
msg += "\nSQL: " + err.sql;
}
if (err.traceback) {
I'm moving down the file to check the remaining contents.
root@b8a745a412a8:/workspace/scubaduck# sed -n '160,320p' scubaduck/static/js/ta
ble_view.js
if (err.traceback) {
msg += "\n" + err.traceback;
}
}
view.innerHTML = `<pre id="error-message">${msg}</pre>`;
document.getElementById("query_info").textContent = "";
}
I’m scrolling to see if the selectedColumns variable is defined in view_settings.
root@b8a745a412a8:/workspace/scubaduck# grep -n "selectedColumns" -n -r scubaduc
k/static/js
scubaduck/static/js/table_view.js:11: let hitsIndex = selectedColumns.indexOf("
Hits");
scubaduck/static/js/table_view.js:17: selectedColumns.forEach((col, i) => {
scubaduck/static/js/table_view.js:51: const col = selectedColumns[i];
scubaduck/static/js/view_settings.js:15:let selectedColumns = [];
scubaduck/static/js/view_settings.js:440: selectedColumns = groupBy.chips.sli
ce();
scubaduck/static/js/view_settings.js:441: if (document.getElementById('show_h
its').checked) selectedColumns.push('Hits');
scubaduck/static/js/view_settings.js:443: if (!selectedColumns.includes(c))
selectedColumns.push(c);
scubaduck/static/js/view_settings.js:446: if (dc.include && !selectedColumn
s.includes(dc.name)) selectedColumns.push(dc.name);
scubaduck/static/js/view_settings.js:449: selectedColumns = base.slice();
scubaduck/static/js/view_settings.js:451: if (dc.include) selectedColumns.p
ush(dc.name);
scubaduck/static/js/view_settings.js:454: columnValues[type] = selectedColumns.
slice();
scubaduck/static/js/view_settings.js:456: if (orderCol && !selectedColumns.incl
udes(orderCol)) {
scubaduck/static/js/view_settings.js:457: selectedColumns.push(orderCol);
scubaduck/static/js/view_settings.js:692: columns: selectedColumns.filter(c =
>
scubaduck/static/js/timeseries_chart.js:38: const valueCols = selectedColumns.s
lice(groups.length + hasHits);
I’ll open view_settings.js to check how the columns are being selected.
ew_settings.js
}
function updateSelectedColumns(type = graphTypeSel.value) {
const base = allColumns.filter(name => {
const cb = document.querySelector(`#column_groups input[value="${name}"]`);
if (!cb || !cb.checked) return false;
if (type === 'table' && isStringColumn(name)) return false;
return true;
});
if (type === 'table' || type === 'timeseries') {
selectedColumns = groupBy.chips.slice();
if (document.getElementById('show_hits').checked) selectedColumns.push('Hits
');
base.forEach(c => {
if (!selectedColumns.includes(c)) selectedColumns.push(c);
});
derivedColumns.forEach(dc => {
if (dc.include && !selectedColumns.includes(dc.name)) selectedColumns.push
(dc.name);
});
} else {
selectedColumns = base.slice();
derivedColumns.forEach(dc => {
if (dc.include) selectedColumns.push(dc.name);
});
}
columnValues[type] = selectedColumns.slice();
const orderCol = document.getElementById('order_by').value;
if (orderCol && !selectedColumns.includes(orderCol)) {
selectedColumns.push(orderCol);
}
updateColumnsTabCount();
}
function isStringColumn(name) {
const t = (columnTypes[name] || '').toUpperCase();
return t.includes('CHAR') || t.includes('STRING') || t.includes('VARCHAR');
}
function isIntegerColumn(name) {
const t = (columnTypes[name] || '').toUpperCase();
return t.includes('INT');
}
function isTimeColumn(name) {
const t = (columnTypes[name] || '').toUpperCase();
if (t.includes('TIMESTAMP') || t.includes('DATE') || t.includes('TIME')) retur
n true;
const sel = document.getElementById('time_column').value;
const xsel = document.getElementById('x_axis').value;
if (name === sel || name === xsel) return true;
return false;
}
function formatNumber(val) {
if (typeof val !== 'number') val = Number(val);
if (Number.isNaN(val)) return '';
if (val === 0) return '0';
const abs = Math.abs(val);
if (abs > 999.999) {
const units = [
{n: 1e12, s: 'T'},
{n: 1e9, s: 'B'},
{n: 1e6, s: 'M'},
{n: 1e3, s: 'K'},
];
for (const u of units) {
if (abs >= u.n) {
return (val / u.n).toFixed(2) + ' ' + u.s;
}
}
}
if (abs < 0.0005) return '0.000';
if (Number.isInteger(val)) return val.toString();
return val.toFixed(3);
}
function addFilter() {
const container = document.createElement('div');
container.className = 'filter';
container.innerHTML = `
<div class="filter-row">
<select class="f-col"></select>
<select class="f-op"></select>
<button type="button" class="remove" onclick="this.closest('.filter').remo
ve()">✖</button>
</div>
<div class="chip-box">
<div class="chip-input">
<input class="f-val" type="text">
<button type="button" class="chip-copy">⎘</button>
</div>
<div class="chip-dropdown"></div>
</div>
`;
const colSel = container.querySelector('.f-col');
colSel.innerHTML = allColumns.map(c => `<option value="${c}">${c}</option>`).j
oin('');
initDropdown(colSel);
function populateOps() {
const opSel = container.querySelector('.f-op');
const col = colSel.value;
const ops = isStringColumn(col)
? [
['=', '='],
['!=', '!='],
['~', 'matches regex'],
['!~', 'not matches regex'],
['contains', 'contains'],
['!contains', 'not contains'],
['empty', 'empty'],
['!empty', 'not empty'],
['LIKE', 'like'],
]
: [
['=', '='],
['!=', '!='],
['<', '<'],
['>', '>'],
];
opSel.innerHTML = ops.map(o => `<option value="${o[0]}">${o[1]}</option>`).j
...
>
<textarea class="d-expr" rows="2"></textarea>
`;
document.getElementById('derived_list').appendChild(container);
const obj = {
type: data.type || 'string',
name: data.name || nextDerivedName(),
expr: data.expr || '',
include: data.include !== undefined ? data.include : true,
el: container
};
container.querySelector('.d-type').value = obj.type;
container.querySelector('.d-name').value = obj.name;
container.querySelector('.d-expr').value = obj.expr;
container.querySelector('.d-use').checked = obj.include;
['change','input'].forEach(evt => {
container.addEventListener(evt, refreshDerivedColumns);
});
derivedColumns.push(obj);
refreshDerivedColumns();
}
function removeDerived(btn) {
const el = btn.closest('.derived');
const idx = derivedColumns.findIndex(d => d.el === el);
if (idx !== -1) {
derivedColumns.splice(idx, 1);
}
el.remove();
refreshDerivedColumns();
}
function refreshDerivedColumns() {
allColumns.splice(0, allColumns.length, ...baseColumns);
stringColumns.splice(0, stringColumns.length, ...baseStringColumns);
integerColumns.splice(0, integerColumns.length, ...baseIntegerColumns);
timeColumns.splice(0, timeColumns.length, ...baseTimeColumns);
timeColumnOptions.splice(0, timeColumnOptions.length, ...baseTimeColumnOptions
);
derivedColumns.forEach(d => {
d.type = d.el.querySelector('.d-type').value;
d.name = d.el.querySelector('.d-name').value;
d.expr = d.el.querySelector('.d-expr').value;
d.include = d.el.querySelector('.d-use').checked;
allColumns.push(d.name);
columnTypes[d.name] = d.type;
if (d.type === 'string') {
stringColumns.push(d.name);
} else {
integerColumns.push(d.name);
timeColumnOptions.push(d.name);
}
});
updateSelectedColumns();
}
let lastQueryTime = 0;
let queryStart = 0;
function dive(push=true) {
const params = collectParams();
if (push) {
history.pushState(params, '', paramsToSearch(params));
}
const payload = Object.assign({}, params);
const dcMap = {};
(params.derived_columns || []).forEach(d => {
if (d.include) dcMap[d.name] = d.expr;
});
payload.derived_columns = dcMap;
const view = document.getElementById('view');
view.innerHTML = '<p>Loading...</p>';
window.lastResults = undefined;
queryStart = performance.now();
fetch('/api/query', {method:'POST', headers:{'Content-Type':'application/json'
}, body:JSON.stringify(payload)})
.then(async r => {
const data = await r.json();
if (!r.ok) throw data;
return data;
})
.then(data => {
lastQueryTime = Math.round(performance.now() - queryStart);
showResults(data);
})
.catch(err => {
showError(err);
});
}
function collectParams() {
updateSelectedColumns();
const payload = {
table: document.getElementById('table').value,
time_column: document.getElementById('time_column').value,
time_unit: document.getElementById('time_unit').value,
start: document.getElementById('start').value,
end: document.getElementById('end').value,
order_by: document.getElementById('order_by').value,
order_dir: orderDir,
limit: parseInt(document.getElementById('limit').value, 10),
columns: selectedColumns.filter(c =>
c !== 'Hits' && !derivedColumns.some(dc => dc.name === c)
),
samples_columns: columnValues.samples.slice(),
table_columns: columnValues.table.slice(),
timeseries_columns: columnValues.timeseries.slice(),
graph_type: graphTypeSel.value,
filters: Array.from(document.querySelectorAll('#filters .filter')).map(f =>
{
const chips = f.chips || [];
const op = f.querySelector('.f-op').value;
let value = null;
if (op !== 'empty' && op !== '!empty') {
value = chips.length === 0 ? null : (chips.length === 1 ? chips[0] : chi
ps);
}
const colSel = f.querySelector('.f-col select') || f.querySelector('.f-col
');
return {column: colSel.value, op, value};
}),
derived_columns: Array.from(document.querySelectorAll('#derived_list .derive
d')).map(d => ({
type: d.querySelector('.d-type').value,
I’m paging towards the 670th line to verify the updateSelectedColumns and related updates.
root@b8a745a412a8:/workspace/scubaduck# grep -n "show_hits" -n scubaduck/static/
js/view_settings.js
149: document.getElementById('show_hits_field').style.display = showTable ? 'fl
ex' : 'none';
441: if (document.getElementById('show_hits').checked) selectedColumns.push('
Hits');
719: payload.show_hits = document.getElementById('show_hits').checked;
749: if (params.show_hits) sp.set('show_hits', '1');
792: document.getElementById('show_hits').checked = params.show_hits ?? true;
849: if (sp.has('show_hits')) params.show_hits = sp.get('show_hits') === '1';
I need to find where the count aggregator is located, probably in static files.
root@b8a745a412a8:/workspace/scubaduck# grep -n "aggregate" -n scubaduck/static/
js -n | head
grep: scubaduck/static/js: Is a directory
I made a mistake with the command. Let me try running it again.
root@b8a745a412a8:/workspace/scubaduck# grep -n "aggregate" scubaduck/static/js
-r | head
scubaduck/static/js/table_view.js:25: const agg = document.getElementById("
aggregate").value.toLowerCase();
scubaduck/static/js/view_settings.js:148: document.getElementById('aggregate_fi
eld').style.display = showTable || showTS ? 'flex' : 'none';
scubaduck/static/js/view_settings.js:345: initDropdown(document.getElementByI
d('aggregate'));
scubaduck/static/js/view_settings.js:586: <option value="aggregated">Aggr
egated</option>
scubaduck/static/js/view_settings.js:718: payload.aggregate = document.getEle
mentById('aggregate').value;
scubaduck/static/js/view_settings.js:748: if (params.aggregate) sp.set('aggre
gate', params.aggregate);
scubaduck/static/js/view_settings.js:791: if (params.aggregate) document.getEle
mentById('aggregate').value = params.aggregate;
scubaduck/static/js/view_settings.js:848: if (sp.has('aggregate')) params.aggre
gate = sp.get('aggregate');
scubaduck/static/js/timeseries_chart.js:313: const agg = document.getElementB
yId('aggregate').value.toLowerCase();
When aggregate is count, I shouldn't show headers for non-grouped columns, just "Hits".
With count aggregate, only group_by and Hits columns should be shown, not others.
root@b8a745a412a8:/workspace/scubaduck# sed -n '1,160p' scubaduck/static/js/view
_settings.js
// Logic for View Settings, Columns, and URL handling extracted from index.html
const allColumns = [];
const baseColumns = [];
const columnTypes = {};
const stringColumns = [];
const baseStringColumns = [];
const integerColumns = [];
const baseIntegerColumns = [];
const timeColumns = [];
const baseTimeColumns = [];
const timeColumnOptions = [];
const baseTimeColumnOptions = [];
const derivedColumns = [];
let selectedColumns = [];
let displayType = 'samples';
let groupBy = {chips: [], addChip: () => {}, renderChips: () => {}};
let defaultTimeColumn = '';
const limitInput = document.getElementById('limit');
const limitValues = {
samples: parseInt(limitInput.value, 10),
table: parseInt(limitInput.value, 10),
timeseries: 7
};
const columnValues = {
samples: [],
table: [],
timeseries: []
};
limitInput.addEventListener('input', () => {
limitValues[displayType] = parseInt(limitInput.value, 10);
limitInput.dataset.setByUser = '1';
});
function initDropdown(select) {
// Avoid creating duplicate wrappers if this dropdown was already initialised.
if (select.dataset.dropdownInit) {
const disp = select.parentElement?.querySelector('.dropdown-display');
if (disp) {
const opt = select.options[select.selectedIndex];
disp.textContent = opt ? opt.textContent : '';
}
return;
}
select.dataset.dropdownInit = '1';
const wrapper = document.createElement('div');
wrapper.className = 'dropdown';
if (select.classList.contains('f-col')) {
wrapper.classList.add('f-col');
}
select.parentNode.insertBefore(wrapper, select);
wrapper.appendChild(select);
select.style.display = 'none';
const disp = document.createElement('div');
disp.className = 'dropdown-display';
function updateDisplay() {
const opt = select.options[select.selectedIndex];
disp.textContent = opt ? opt.textContent : '';
}
updateDisplay();
wrapper.appendChild(disp);
const menu = document.createElement('div');
menu.className = 'dropdown-menu';
const search = document.createElement('input');
search.placeholder = 'Search';
menu.appendChild(search);
const list = document.createElement('div');
menu.appendChild(list);
wrapper.appendChild(menu);
function close() {
menu.style.display = 'none';
}
function open() {
renderOptions();
menu.style.display = 'block';
search.focus();
}
disp.addEventListener('click', () => {
if (menu.style.display === 'block') {
close();
} else {
open();
}
});
document.addEventListener('click', e => {
if (!wrapper.contains(e.target)) {
close();
}
});
function renderOptions() {
const q = search.value.toLowerCase();
list.innerHTML = '';
Array.from(select.options).forEach(o => {
if (!o.textContent.toLowerCase().includes(q)) return;
const div = document.createElement('div');
div.className = 'option';
if (q) {
const text = o.textContent;
const idx = text.toLowerCase().indexOf(q);
if (idx !== -1) {
div.innerHTML =
text.slice(0, idx) +
'<u>' +
text.slice(idx, idx + q.length) +
'</u>' +
text.slice(idx + q.length);
} else {
div.textContent = text;
}
} else {
div.textContent = o.textContent;
}
if (o.value === select.value) div.classList.add('selected');
div.addEventListener('mousedown', evt => {
evt.preventDefault();
select.value = o.value;
select.dispatchEvent(new Event('change'));
updateDisplay();
close();
});
list.appendChild(div);
});
}
search.addEventListener('input', renderOptions);
select.addEventListener('change', updateDisplay);
}
let orderDir = 'ASC';
const orderDirBtn = document.getElementById('order_dir');
const graphTypeSel = document.getElementById('graph_type');
function updateOrderDirButton() {
orderDirBtn.textContent = orderDir + (orderDir === 'ASC' ? ' \u25B2' : ' \u25B
C');
}
function updateDisplayTypeUI() {
const prevType = displayType;
updateSelectedColumns(prevType);
const newType = graphTypeSel.value;
const showTable = newType === 'table';
const showTS = newType === 'timeseries';
document.getElementById('group_by_field').style.display = showTable || showTS
? 'flex' : 'none';
document.getElementById('aggregate_field').style.display = showTable || showTS
? 'flex' : 'none';
document.getElementById('show_hits_field').style.display = showTable ? 'flex'
: 'none';
document.getElementById('x_axis_field').style.display = showTS ? 'flex' : 'non
e';
document.getElementById('granularity_field').style.display = showTS ? 'flex' :
'none';
document.getElementById('fill_field').style.display = showTS ? 'flex' : 'none'
;
document.querySelectorAll('#column_groups .col-group').forEach(g => {
if (g.querySelector('.col-group-header').textContent.startsWith('Strings'))
{
g.style.display = showTable || showTS ? 'none' : '';
}
});
limitValues[prevType] = parseInt(limitInput.value, 10);
if (showTS && limitValues.timeseries === undefined) {
limitValues.timeseries = 7;
root@b8a745a412a8:/workspace/scubaduck# sed -n '320,480p' scubaduck/static/js/vi
ew_settings.js
ul.querySelectorAll('input').forEach(cb => (cb.checked = false));
updateSelectedColumns();
});
div.appendChild(ul);
groupsEl.appendChild(div);
});
document.getElementById('columns_all').addEventListener('click', e => {
e.preventDefault();
groupsEl.querySelectorAll('input').forEach(cb => (cb.checked = true));
updateSelectedColumns();
});
document.getElementById('columns_none').addEventListener('click', e => {
e.preventDefault();
groupsEl.querySelectorAll('input').forEach(cb => (cb.checked = false));
updateSelectedColumns();
});
updateSelectedColumns();
columnValues.samples = allColumns.slice();
columnValues.table = [];
columnValues.timeseries = [];
groupBy = document.getElementById('group_by').closest('.field');
initChipInput(groupBy, typed =>
allColumns.filter(c => c.toLowerCase().includes(typed.toLowerCase()))
);
initDropdown(orderSelect);
initDropdown(document.getElementById('aggregate'));
});
}
let columnsInitialized = false;
fetch('/api/tables').then(r => r.json()).then(tables => {
const tableSel = document.getElementById('table');
tables.forEach(t => {
const o = document.createElement('option');
o.value = t;
o.textContent = t;
tableSel.appendChild(o);
});
initDropdown(tableSel);
const measure = document.createElement('span');
measure.style.visibility = 'hidden';
measure.style.position = 'absolute';
document.body.appendChild(measure);
let maxWidth = 0;
tables.forEach(t => {
measure.textContent = t;
const w = measure.getBoundingClientRect().width;
if (w > maxWidth) maxWidth = w;
});
measure.remove();
const disp = tableSel.parentElement.querySelector('.dropdown-display');
if (disp) disp.style.minWidth = maxWidth + 30 + 'px';
const table = parseSearch().table || tables[0];
tableSel.value = table;
tableSel.dispatchEvent(new Event('change'));
loadColumns(table).then(() => {
updateDisplayTypeUI();
addFilter();
initFromUrl();
columnsInitialized = true;
});
tableSel.addEventListener('change', () => {
loadColumns(tableSel.value).then(() => {
if (columnsInitialized) {
// keep the newly selected table when updating from the URL
const params = parseSearch();
params.table = tableSel.value;
applyParams(params);
}
});
});
});
document.querySelectorAll('#tabs .tab').forEach(btn => {
btn.addEventListener('click', () => {
document.querySelectorAll('#tabs .tab').forEach(t => t.classList.remove('act
ive'));
document.querySelectorAll('.tab-content').forEach(c => c.classList.remove('a
ctive'));
btn.classList.add('active');
document.getElementById(btn.dataset.tab).classList.add('active');
});
});
document.querySelectorAll('.rel-btn').forEach(btn => {
btn.addEventListener('click', () => {
const dd = document.getElementById(btn.dataset.target);
const show = dd.style.display === 'none' || dd.style.display === '';
document.querySelectorAll('.rel-dropdown').forEach(d => (d.style.display = '
none'));
dd.style.display = show ? 'block' : 'none';
});
});
document.querySelectorAll('.rel-dropdown div').forEach(opt => {
opt.addEventListener('click', () => {
const box = opt.closest('.rel-box');
const input = box.querySelector('input');
input.value = opt.dataset.value || opt.textContent;
opt.parentElement.style.display = 'none';
});
});
document.addEventListener('click', e => {
document.querySelectorAll('.rel-dropdown').forEach(dd => {
if (!dd.parentElement.contains(e.target)) dd.style.display = 'none';
});
});
function updateColumnsTabCount() {
const baseCount = document.querySelectorAll('#column_groups input:checked').le
ngth;
const derivedCount = document.querySelectorAll('#derived_list .derived .d-use:
checked').length;
const btn = document.getElementById('columns_tab');
if (btn) btn.textContent = `Columns (${baseCount + derivedCount})`;
}
function updateSelectedColumns(type = graphTypeSel.value) {
const base = allColumns.filter(name => {
const cb = document.querySelector(`#column_groups input[value="${name}"]`);
if (!cb || !cb.checked) return false;
if (type === 'table' && isStringColumn(name)) return false;
return true;
});
if (type === 'table' || type === 'timeseries') {
selectedColumns = groupBy.chips.slice();
if (document.getElementById('show_hits').checked) selectedColumns.push('Hits
');
base.forEach(c => {
if (!selectedColumns.includes(c)) selectedColumns.push(c);
});
derivedColumns.forEach(dc => {
if (dc.include && !selectedColumns.includes(dc.name)) selectedColumns.push
(dc.name);
});
} else {
selectedColumns = base.slice();
derivedColumns.forEach(dc => {
if (dc.include) selectedColumns.push(dc.name);
});
}
columnValues[type] = selectedColumns.slice();
const orderCol = document.getElementById('order_by').value;
if (orderCol && !selectedColumns.includes(orderCol)) {
selectedColumns.push(orderCol);
}
updateColumnsTabCount();
}
function isStringColumn(name) {
const t = (columnTypes[name] || '').toUpperCase();
return t.includes('CHAR') || t.includes('STRING') || t.includes('VARCHAR');
}
function isIntegerColumn(name) {
const t = (columnTypes[name] || '').toUpperCase();
return t.includes('INT');
}
function isTimeColumn(name) {
const t = (columnTypes[name] || '').toUpperCase();
if (t.includes('TIMESTAMP') || t.includes('DATE') || t.includes('TIME')) retur
n true;
const sel = document.getElementById('time_column').value;
const xsel = document.getElementById('x_axis').value;
if (name === sel || name === xsel) return true;
return false;
}
I'll check the test_server_table group_by_count to review the server-side behavior first.
root@b8a745a412a8:/workspace/scubaduck# grep -n "table" -n tests/test_server.py
| head
I'll check tests/test_server_basic to inspect the table tests.
from scubaduck import server
def test_basic_query() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-02 00:00:00",
"order_by": "timestamp",
"order_dir": "ASC",
"limit": 10,
"columns": ["timestamp", "event", "value", "user"],
"filters": [],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert data
rows = data["rows"]
# We expect first three rows (until 2024-01-02 00:00:00)
assert len(rows) == 3
assert rows[0][1] == "login"
assert rows[1][1] == "logout"
def test_js_served() -> None:
app = server.app
client = app.test_client()
rv = client.get("/js/chip_input.js")
assert rv.status_code == 200
assert b"initChipInput" in rv.data
def test_filter_multi_token() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-02 03:00:00",
"order_by": "timestamp",
"limit": 10,
"columns": ["timestamp", "event", "value", "user"],
"filters": [{"column": "user", "op": "=", "value": ["alice", "charlie"]}
],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert data
rows = data["rows"]
# Should only return rows for alice and charlie
assert len(rows) == 3
assert rows[0][3] == "alice"
assert rows[-1][3] == "charlie"
def test_empty_filter_is_noop() -> None:
app = server.app
client = app.test_client()
base_payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"limit": 100,
"columns": ["timestamp", "event", "value", "user"],
}
no_filter = {**base_payload, "filters": []}
empty_filter = {
**base_payload,
"filters": [{"column": "user", "op": "=", "value": None}],
}
rv1 = client.post(
"/api/query", data=json.dumps(no_filter), content_type="application/json
"
)
rv2 = client.post(
"/api/query", data=json.dumps(empty_filter), content_type="application/j
son"
)
rows1 = rv1.get_json()["rows"]
rows2 = rv2.get_json()["rows"]
assert rows1 == rows2
def test_select_columns() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"order_by": "timestamp",
"limit": 10,
"columns": ["timestamp", "user"],
"filters": [],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert data
rows = data["rows"]
assert len(rows[0]) == 2
assert rows[0][1] == "alice"
def test_string_filter_ops() -> None:
app = server.app
client = app.test_client()
base = {
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"order_by": "timestamp",
"limit": 100,
"columns": ["timestamp", "event", "value", "user"],
}
contains = {
**base,
"filters": [{"column": "user", "op": "contains", "value": "ali"}],
}
rv = client.post(
"/api/query", data=json.dumps(contains), content_type="application/json"
)
rows = rv.get_json()["rows"]
assert all("ali" in r[3] for r in rows)
regex = {
**base,
"filters": [{"column": "user", "op": "~", "value": "^a.*"}],
}
rv = client.post(
"/api/query", data=json.dumps(regex), content_type="application/json"
)
rows = rv.get_json()["rows"]
assert all(r[3].startswith("a") for r in rows)
assert len(rows) == 2
not_empty = {**base, "filters": [{"column": "user", "op": "!empty"}]}
rv = client.post(
"/api/query", data=json.dumps(not_empty), content_type="application/json
"
)
assert len(rv.get_json()["rows"]) == 4
def test_order_by_ignored_when_not_selected() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"order_by": "value",
"columns": ["timestamp"],
}
I'm moving forward to check the rest.
root@b8a745a412a8:/workspace/scubaduck# sed -n '160,320p' tests/test_server_basi
c.py
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert "ORDER BY" not in data["sql"]
I'm looking at tests/test_server_time or timeseries to check for queries aggregation.
tests/web_utils.py:23: aggregate: str | None = None,
tests/web_utils.py:45: if aggregate is not None:
tests/web_utils.py:47: select_value(page, "#aggregate", aggregate)
tests/test_server_timeseries.py:21: "aggregate": "Sum",
tests/test_server_timeseries.py:45: "aggregate": "Avg",
tests/test_server_timeseries.py:71: "aggregate": "Count",
tests/test_server_timeseries.py:142: "aggregate": "Count",
tests/test_server_timeseries.py:241: "aggregate": "Avg",
tests/test_web_graph.py:13: assert page.is_visible("#aggregate_field")
tests/test_web_graph.py:24: assert page.is_visible("#aggregate_field")
import json
from typing import Any, cast
from scubaduck import server
def test_group_by_table() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "table",
"order_by": "user",
"limit": 10,
"columns": ["value"],
"group_by": ["user"],
"aggregate": "Sum",
"show_hits": True,
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
rows = rv.get_json()["rows"]
assert rows[0][0] == "alice"
assert rows[0][1] == 2
assert rows[0][2] == 40
def test_table_avg_with_timestamp() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "table",
"order_by": "user",
"limit": 100,
"columns": ["user", "timestamp", "value"],
"group_by": ["user"],
"aggregate": "Avg",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert "error" not in data
rows = data["rows"]
assert rows[0][0] == "alice"
from dateutil import parser
ts = parser.parse(rows[0][1]).replace(tzinfo=None)
assert ts == parser.parse("2024-01-01 12:00:00")
def test_timeseries_basic() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "timeseries",
"limit": 100,
"group_by": ["user"],
"aggregate": "Count",
"columns": ["value"],
"x_axis": "timestamp",
"granularity": "1 day",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert len(data["rows"]) == 4
def test_timeseries_orders_by_xaxis() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "timeseries",
"limit": 100,
"columns": ["value"],
"x_axis": "timestamp",
"granularity": "1 day",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
rows = data["rows"]
from dateutil import parser
timestamps = [parser.parse(r[0]).replace(tzinfo=None) for r in rows]
assert timestamps == sorted(timestamps)
def test_timeseries_count_no_columns() -> None:
app = server.app
client = app.test_client()
payload: dict[str, Any] = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "timeseries",
"granularity": "1 day",
"columns": [],
}
rv = client.post(
I'm continuing to scroll to check for tests related to table count or similar things.
data = rv.get_json()
assert rv.status_code == 200
rows = data["rows"]
assert len(rows) == 2
assert rows[0][1] == 2
assert rows[1][1] == 2
def test_timeseries_limit_applies_to_series() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "timeseries",
"limit": 1,
"order_by": "user",
"group_by": ["user"],
"aggregate": "Count",
"columns": ["value"],
"x_axis": "timestamp",
"granularity": "1 day",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert len(data["rows"]) == 2
assert all(r[1] == "alice" for r in data["rows"])
def test_timeseries_auto_and_fine_buckets() -> None:
app = server.app
client = app.test_client()
def run(gran: str) -> None:
payload = {
"start": "2024-01-01 00:00:00",
"end": "2024-01-02 03:00:00",
"graph_type": "timeseries",
"columns": ["value"],
"x_axis": "timestamp",
"granularity": gran,
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/js
on"
)
data = rv.get_json()
assert rv.status_code == 200
from dateutil import parser
start = parser.parse(cast(str, payload["start"])).replace(tzinfo=None)
buckets = [
parser.parse(cast(str, r[0])).replace(tzinfo=None) for r in data["ro
ws"]
]
assert buckets[0] == start
if len(buckets) > 1:
step = (buckets[1] - buckets[0]).total_seconds()
assert step % data["bucket_size"] == 0
assert any(r[1] != 0 for r in data["rows"])
run("Auto")
run("Fine")
def test_timeseries_string_column_error() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "timeseries",
"limit": 7,
"columns": ["timestamp", "event", "value", "user"],
"x_axis": "timestamp",
"granularity": "1 hour",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
assert rv.status_code == 200
def test_derived_column_basic() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"limit": 10,
"columns": ["timestamp"],
"derived_columns": {"val2": "value * 2"},
"filters": [],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert data["rows"][0][1] == 20
def test_timeseries_derived_column() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "timeseries",
"granularity": "1 hour",
"limit": 7,
"columns": ["value"],
"derived_columns": {"derived_1": "value * 2"},
root@b8a745a412a8:/workspace/scubaduck# grep -n "table" -r tests | grep -i count
| head
I'll search for "Hits" and "table dimensions" among the tests to find relevant checks.
root@b8a745a412a8:/workspace/scubaduck# grep -n "Hits" -r tests | head
tests/test_web_table.py:234: assert "Hits" in headers
tests/test_server_errors.py:57: "columns": ["user", "Hits", "value"],
I'll inspect tests/test_web_table around line 230 to check header show/hide functionality.
ts = parser.parse(row[1]).replace(tzinfo=None)
assert ts == parser.parse("2024-01-01 13:00:00")
assert row[2] == 25
def test_table_headers_show_aggregate(page: Any, server_url: str) -> None:
run_query(
page,
server_url,
aggregate="Avg",
)
headers = page.locator("#results th").all_inner_texts()
assert "Hits" in headers
assert "timestamp (avg)" in headers
assert "value (avg)" in headers
def test_format_number_function(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
vals = page.evaluate(
"() => [formatNumber(815210), formatNumber(999.999), formatNumber(0.0004
), formatNumber(0)]"
)
assert vals == ["815.21 K", "999.999", "0.000", "0"]
def test_numeric_cell_nowrap(page: Any, server_url: str) -> None:
run_query(page, server_url, limit=10)
whitespace = page.evaluate(
"getComputedStyle(document.querySelector('#results td:nth-child(3)')).wh
iteSpace"
)
assert whitespace == "nowrap"
def test_date_cell_nowrap(page: Any, server_url: str) -> None:
run_query(page, server_url, limit=10)
whitespace = page.evaluate(
"getComputedStyle(document.querySelector('#results td:nth-child(1)')).wh
iteSpace"
)
I’ll keep going after 220 lines to check for more test cases.
def test_derived_column_query(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
page.click("text=Columns")
page.click("text=Add Derived")
expr = page.query_selector("#derived_list .derived textarea")
assert expr
expr.fill("value * 2")
page.click("text=View Settings")
page.fill("#start", "2024-01-01 00:00:00")
page.fill("#end", "2024-01-03 00:00:00")
page.fill("#limit", "10")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
data = page.evaluate("window.lastResults")
assert data["rows"][0][-1] == 20
def test_derived_column_remove(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
page.click("text=Columns")
page.click("text=Add Derived")
assert page.query_selector("#derived_list .derived button.remove")
page.click("#derived_list .derived button.remove")
count = page.evaluate("document.querySelectorAll('#derived_list .derived').l
ength")
assert count == 0
def test_sql_query_display(page: Any, server_url: str) -> None:
data = run_query(
page,
server_url,
start="2024-01-01 00:00:00",
end="2024-01-02 00:00:00",
order_by="timestamp",
limit=10,
)
sql = data["sql"]
displayed = page.text_content("#sql_query")
assert displayed is not None
assert displayed.strip() == sql
I'm checking the static dataset to see if it includes an 'id' column for testing.
root@b8a745a412a8:/workspace/scubaduck# head scubaduck/sample.csv
timestamp,event,value,user
2024-01-01 00:00:00,login,10,alice
2024-01-01 01:00:00,logout,20,bob
2024-01-02 00:00:00,login,30,alice
2024-01-02 03:00:00,login,40,charlie
This isn't the test one; I'll check the "TEST" dataset built from _create_test_database.
uck/server.py
82:def _create_test_database() -> duckdb.DuckDBPyConnection:
404: con = _create_test_database()
def _create_test_database() -> duckdb.DuckDBPyConnection:
"""Return a DuckDB connection with a small multi-table dataset."""
con = duckdb.connect()
con.execute(
"CREATE TABLE events (id INTEGER PRIMARY KEY, ts INTEGER, val REAL, name
TEXT, flag BOOLEAN)"
)
con.execute("INSERT INTO events VALUES (1, 1704067200, 1.5, 'alice', 1)")
con.execute("INSERT INTO events VALUES (2, 1704070800, 2.0, 'bob', 0)")
con.execute('CREATE TABLE extra (ts INTEGER, "desc" TEXT, num INTEGER)')
con.execute("INSERT INTO extra VALUES (1704067200, 'x', 1)")
con.execute("INSERT INTO extra VALUES (1704070800, 'y', 2)")
return con
_REL_RE = re.compile(
r"([+-]?\d+(?:\.\d*)?)\s*(hour|hours|day|days|week|weeks|fortnight|fortnight
s|month|months|year|years)",
re.IGNORECASE,
)
def parse_time(val: str | None) -> str | None:
"""Parse an absolute or relative time string into ``YYYY-MM-DD HH:MM:SS``.""
"
if val is None or val == "":
return None
s = val.strip()
if s.lower() == "now":
dt = datetime.now(timezone.utc)
return dt.replace(microsecond=0).strftime("%Y-%m-%d %H:%M:%S")
m = _REL_RE.fullmatch(s)
if m:
qty = float(m.group(1))
unit = m.group(2).lower()
now = datetime.now(timezone.utc)
dt: datetime
if unit.startswith("hour"):
dt = now + timedelta(hours=qty)
elif unit.startswith("day"):
dt = now + timedelta(days=qty)
elif unit.startswith("week"):
dt = now + timedelta(weeks=qty)
elif unit.startswith("fortnight"):
dt = now + timedelta(weeks=2 * qty)
elif unit.startswith("month"):
if qty.is_integer():
dt = now + relativedelta(months=int(qty))
else:
dt = now + timedelta(days=30 * qty)
elif unit.startswith("year"):
if qty.is_integer():
dt = now + relativedelta(years=int(qty))
else:
dt = now + timedelta(days=365 * qty)
else: # pragma: no cover - defensive
raise ValueError(f"Unsupported unit: {unit}")
return dt.replace(microsecond=0).strftime("%Y-%m-%d %H:%M:%S")
dt = dtparser.parse(s)
return dt.replace(microsecond=0, tzinfo=None).strftime("%Y-%m-%d %H:%M:%S")
def _numeric_to_datetime(value: int | float, unit: str) -> datetime:
"""Convert a numeric timestamp ``value`` with unit ``unit`` to ``datetime``.
Heuristically fall back to seconds when the converted value is before 1990
but the seconds interpretation is in a reasonable range. This handles
integer columns stored in seconds even when ``unit`` is mistakenly set to a
finer granularity.
"""
divisor = {
"s": 1,
"ms": 1000,
"us": 1_000_000,
"ns": 1_000_000_000,
}.get(unit, 1)
dt = datetime.fromtimestamp(int(value) / divisor, tz=timezone.utc)
if unit != "s" and dt.year < 1990:
I see the dataset "TEST" is in use. I'll check tests/test_web_table and conftest.py.
from __future__ import annotations
import threading
from collections.abc import Iterator
import pytest
from werkzeug.serving import make_server
from scubaduck.server import app
@pytest.fixture()
def server_url() -> Iterator[str]:
httpd = make_server("127.0.0.1", 0, app)
port = httpd.server_port
thread = threading.Thread(target=httpd.serve_forever)
thread.start()
try:
yield f"http://127.0.0.1:{port}"
finally:
httpd.shutdown()
thread.join()
I’m going to open scubaduck/init.py to examine the server fixture.
root@b8a745a412a8:/workspace/scubaduck# cat scubaduck/__init__.py
def hello() -> str:
return "Hello from scubaduck!"
I need to look for where "app" is defined in the code.
root@b8a745a412a8:/workspace/scubaduck# grep -n "create_app" -n scubaduck -r
scubaduck/server.py:397:def create_app(db_file: str | Path | None = None) -> Fla
sk:
scubaduck/server.py:720:app = create_app()
except Exception as exc:
tb = traceback.format_exc()
print(f"Query failed:\n{sql}\n{tb}")
return (
jsonify({"sql": sql, "error": str(exc), "traceback": tb}),
400,
)
result: Dict[str, Any] = {"sql": sql, "rows": rows}
if params.start is not None:
result["start"] = str(params.start)
if params.end is not None:
result["end"] = str(params.end)
if bucket_size is not None:
result["bucket_size"] = bucket_size
return jsonify(result)
return app
app = create_app()
if __name__ == "__main__":
app.run(debug=True)
I think it might be in the SCUBADUCK_DB
variable or set during tests.
root@b8a745a412a8:/workspace/scubaduck# cat codex_setup.sh
uv sync --frozen
source .venv/bin/activate
python -c "import os; import duckdb; con = duckdb.connect(); con.execute(f\"SET
http_proxy = '{os.getenv(\"HTTP_PROXY\")}'\"); con.execute(\"INSTALL 'sqlite';\"
)"
playwright install chromium
echo "source .venv/bin/activate" >> ~/.bashrc
I need to find where the "TEST" dataset is used in test files.
root@b8a745a412a8:/workspace/scubaduck# grep -n "TEST" -r tests | head
tests/test_server_timeseries.py:253: app = server.create_app("TEST")
tests/test_server_time.py:234: app = server.create_app("TEST")
tests/test_server_time.py:252: app = server.create_app("TEST")
tests/test_multi_table_web.py:14: app = create_app("TEST")
I'm opening tests/test_server_timeseries to look for tests using create_app('TEST').
"derived_columns": {"derived_1": "value * 2"},
"aggregate": "Avg",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
rows = data["rows"]
assert all(r[2] == r[1] * 2 for r in rows)
def test_reserved_word_column() -> None:
app = server.create_app("TEST")
client = app.test_client()
payload = {
"table": "extra",
"columns": ["ts", "desc"],
"order_by": "ts",
"time_column": "",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert len(data["rows"]) == 2
assert data["rows"][0][1] == "x"
The test for 'reserved_word_column' uses create_app('TEST'), but not the new feature. I'll check test_server_time.
"table": "events",
"columns": ["timestamp", "event"],
"time_column": "",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert "start" not in data and "end" not in data
assert len(data["rows"]) == 4
def test_test_dataset_int32_time_s() -> None:
app = server.create_app("TEST")
client = app.test_client()
payload = {
"table": "events",
"time_column": "ts",
"time_unit": "s",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert data["start"] == "2024-01-01 00:00:00"
assert data["end"] == "2024-01-01 01:00:00"
assert len(data["rows"]) == 2
def test_test_dataset_int32_time_us() -> None:
app = server.create_app("TEST")
client = app.test_client()
payload = {
"table": "events",
"time_column": "ts",
"time_unit": "us",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert data["start"] == "2024-01-01 00:00:00"
assert data["end"] == "2024-01-01 01:00:00"
assert len(data["rows"]) == 2
root@b8a745a412a8:/workspace/scubaduck# grep -n "aggregate" -n scubaduck/static/
index.html
303: <div id="aggregate_field" class="field" style="display:none;">
305: <select id="aggregate">
.html
<div id="aggregate_field" class="field" style="display:none;">
<label>Aggregate</label>
<select id="aggregate">
<option>Count</option>
<option>Avg</option>
<option>Sum</option>
<option>Min</option>
<option>Max</option>
<option>Count Distinct</option>
<option>p5</option>
<option>p25</option>
<option>p50</option>
<option>p70</option>
<option>p75</option>
<option>p90</option>
<option>p95</option>
<option>p99</option>
<option>p99.9</option>
I'll check other JS functions related to "count" for timeseries before proceeding with the update.
root@b8a745a412a8:/workspace/scubaduck# grep -n "count" -n scubaduck/static/js/*
.js | head
scubaduck/static/js/timeseries_chart.js:164: const count = span / approxMs(
i);
scubaduck/static/js/timeseries_chart.js:165: const score = Math.abs(count -
15);
scubaduck/static/js/timeseries_chart.js:258: function niceTicks(min, max, count
) {
scubaduck/static/js/timeseries_chart.js:260: const step = niceNum(range / Mat
h.max(count - 1, 1), true);
I'll adjust updateSelectedColumns as specified and look at tests/web_utils.py for run_query details.
from typing import Any
def select_value(page: Any, selector: str, value: str) -> None:
page.evaluate(
"arg => setSelectValue(arg.sel, arg.val)",
{"sel": selector, "val": value},
)
def run_query(
page: Any,
url: str,
*,
start: str | None = None,
end: str | None = None,
order_by: str | None = None,
order_dir: str | None = "ASC",
limit: int | None = None,
group_by: list[str] | None = None,
aggregate: str | None = None,
) -> dict[str, Any]:
page.goto(url)
page.wait_for_selector("#order_by option", state="attached")
page.wait_for_selector("#order_dir", state="attached")
page.wait_for_function("window.lastResults !== undefined")
if start is not None:
page.fill("#start", start)
if end is not None:
page.fill("#end", end)
if order_by is not None:
select_value(page, "#order_by", order_by)
if order_dir is not None and order_dir == "DESC":
page.click("#order_dir")
if limit is not None:
page.fill("#limit", str(limit))
if group_by is not None:
select_value(page, "#graph_type", "table")
page.evaluate(
"g => { groupBy.chips = g; groupBy.renderChips(); }",
group_by,
)
if aggregate is not None:
select_value(page, "#graph_type", "table")
select_value(page, "#aggregate", aggregate)
if page.input_value("#graph_type") != "samples":
page.click("text=Columns")
page.wait_for_selector("#column_groups input", state="attached")
if not page.is_checked("#column_groups input[value='value']"):
page.check("#column_groups input[value='value']")
order_col = order_by or page.input_value("#order_by")
if order_col and not page.is_checked(
f"#column_groups input[value='{order_col}']"
):
if page.query_selector(f"#column_groups input[value='{order_col}']")
:
page.check(f"#column_groups input[value='{order_col}']")
page.click("text=View Settings")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
return page.evaluate("window.lastResults")
I need to verify how run_query
works in existing tests and check table dimensions.
root@b8a745a412a8:/workspace/scubaduck# grep -n "results" -n tests/test_web_tabl
e.py | head
20: "getComputedStyle(document.querySelector('#results th')).textAlign"
24: header = page.locator("#results th").nth(3)
27: return page.locator("#results td:nth-child(4)").all_inner_texts()
38: "getComputedStyle(document.querySelector('#results th:nth-child(4)'))
.color"
51: "getComputedStyle(document.querySelector('#results th:nth-child(4)'))
.color"
80: "getComputedStyle(document.querySelector('#results td')).borderStyle"
85: "getComputedStyle(document.querySelector('#results tr:nth-child(2) td
')).backgroundColor"
88: "getComputedStyle(document.querySelector('#results tr:nth-child(3) td
')).backgroundColor"
92: page.hover("#results tr:nth-child(2)")
94: "getComputedStyle(document.querySelector('#results tr:nth-child(2) td
')).backgroundColor"
root@b8a745a412a8:/workspace/scubaduck# grep -n "dimensions" -n tests/test_web_t
able.py
I need to ensure the test checks correct table dimensions, especially considering horizontal overflow.
start="2024-01-01 00:00:00",
end="2024-01-03 00:00:00",
order_by="timestamp",
group_by=["user"],
aggregate="Avg",
)
assert "error" not in data
assert len(data["rows"]) == 3
def test_table_enhancements(page: Any, server_url: str) -> None:
run_query(
page,
server_url,
start="2024-01-01 00:00:00",
end="2024-01-03 00:00:00",
order_by="timestamp",
limit=10,
)
border = page.evaluate(
"getComputedStyle(document.querySelector('#results td')).borderStyle"
)
assert border == "solid"
color1 = page.evaluate(
"getComputedStyle(document.querySelector('#results tr:nth-child(2) td'))
.backgroundColor"
)
color2 = page.evaluate(
"getComputedStyle(document.querySelector('#results tr:nth-child(3) td'))
.backgroundColor"
)
assert color1 != color2
page.hover("#results tr:nth-child(2)")
hover_color = page.evaluate(
"getComputedStyle(document.querySelector('#results tr:nth-child(2) td'))
.backgroundColor"
)
assert hover_color != color1
page.click("#results tr:nth-child(2)")
selected_color = page.evaluate(
"getComputedStyle(document.querySelector('#results tr:nth-child(2) td'))
.backgroundColor"
)
assert "189, 228, 255" in selected_color
overflow = page.evaluate(
"var v=document.getElementById('view'); v.scrollWidth > v.clientWidth"
)
assert not overflow
def test_table_single_selection(page: Any, server_url: str) -> None:
run_query(
page,
server_url,
start="2024-01-01 00:00:00",
end="2024-01-03 00:00:00",
order_by="timestamp",
limit=10,
)
page.click("#results tr:nth-child(2)")
page.click("#results tr:nth-child(3)")
I’ll replicate the ‘table_enhancements’ test and check for header names, row counts, and width.
root@b8a745a412a8:/workspace/scubaduck# grep -n "scrollWidth" -r tests
tests/test_web_timeseries.py:273: "var v=document.getElementById('view');
v.scrollWidth > v.clientWidth"
tests/test_web_table.py:105: "var v=document.getElementById('view'); v.sc
rollWidth > v.clientWidth"
I'll open the timeseries test snippet to replicate the issue.
ies.py
def test_timeseries_no_overflow(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#graph_type", state="attached")
select_value(page, "#graph_type", "timeseries")
page.click("text=Columns")
page.check("#column_groups input[value='value']")
page.click("text=View Settings")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
overflow = page.evaluate(
"var v=document.getElementById('view'); v.scrollWidth > v.clientWidth"
)
assert not overflow
def test_timeseries_axis_ticks(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#graph_type", state="attached")
root@b8a745a412a8:/workspace/scubaduck# grep -n "order_by" -n scubaduck/static/i
ndex.html
251: <select id="order_by"></select>
<div data-value="-1 day">-1 day</div>
<div data-value="-3 days">-3 days</div>
<div data-value="-1 week">-1 week</div>
<div data-value="-1 fortnight">-1 fortnight</div>
<div data-value="-30 days">-30 days</div>
<div data-value="-90 days">-90 days</div>
</div>
</div>
</div>
<div class="field">
<label>Order By<span class="help" title="Choose a column to sort resul
ts by.">[?]</span></label>
<select id="order_by"></select>
<button id="order_dir" type="button">ASC \u25B2</button>
</div>
<div class="field">
<label>Limit<span class="help" title="Choose the maximum number of res
ults to show in the chart after any aggregations have been applied. For example,
a limit of 10 will show no more than 10 rows for a table, etc.">[?]</span></lab
el>
<input id="limit" type="number" value="100" />
</div>
<div id="x_axis_field" class="field" style="display:none;">
<label>X-axis</label>
<select id="x_axis"></select>
I'll look through the JavaScript to find where the order_by options are set.
root@b8a745a412a8:/workspace/scubaduck# grep -n "order_by" scubaduck/static/js -
r | head
scubaduck/static/js/view_settings.js:172: document.getElementById('order_by')
.value = '';
scubaduck/static/js/view_settings.js:198: const orderSelect = document.getEle
mentById('order_by');
scubaduck/static/js/view_settings.js:455: const orderCol = document.getElementB
yId('order_by').value;
scubaduck/static/js/view_settings.js:689: order_by: document.getElementById('
order_by').value,
scubaduck/static/js/view_settings.js:737: if (params.order_by) sp.set('order_by
', params.order_by);
scubaduck/static/js/view_settings.js:767: if (params.order_by) {
scubaduck/static/js/view_settings.js:768: document.getElementById('order_by')
.value = params.order_by;
scubaduck/static/js/view_settings.js:837: if (sp.has('order_by')) params.order_
by = sp.get('order_by');
root@b8a745a412a8:/workspace/scubaduck# sed -n '180,220p' scubaduck/static/js/vi
ew_settings.js
? 'flex'
: 'none';
document.getElementById('end').closest('.field').style.display = show
? 'flex'
: 'none';
document.getElementById('time_unit').style.display = show ? '' : 'none';
}
orderDirBtn.addEventListener('click', () => {
orderDir = orderDir === 'ASC' ? 'DESC' : 'ASC';
updateOrderDirButton();
});
updateOrderDirButton();
graphTypeSel.addEventListener('change', updateDisplayTypeUI);
document.getElementById('time_column').addEventListener('change', updateTimeFiel
dVisibility);
updateTimeFieldVisibility();
function loadColumns(table) {
return fetch('/api/columns?table=' + encodeURIComponent(table)).then(r => r.js
on()).then(cols => {
const orderSelect = document.getElementById('order_by');
const xAxisSelect = document.getElementById('x_axis');
const groupsEl = document.getElementById('column_groups');
const timeColumnSelect = document.getElementById('time_column');
orderSelect.innerHTML = '';
xAxisSelect.innerHTML = '';
const defOpt = document.createElement('option');
defOpt.value = '';
defOpt.textContent = '(default)';
xAxisSelect.appendChild(defOpt);
timeColumnSelect.innerHTML = '';
const noneOpt = document.createElement('option');
noneOpt.value = '';
noneOpt.textContent = '(none)';
timeColumnSelect.appendChild(noneOpt);
groupsEl.innerHTML = '';
allColumns.length = 0;
stringColumns.length = 0;
integerColumns.length = 0;
timeColumns.length = 0;
timeColumnOptions.length = 0;
baseColumns.length = 0;
baseStringColumns.length = 0;
I'm reviewing how updateSelectedColumns
handles checkbox groups and order_by edge cases. I’ll update tests for count aggregation.
root@b8a745a412a8:/workspace/scubaduck# grep -n "timeseries_count_no_columns" -n
tests/test_server_timeseries.py
109:def test_timeseries_count_no_columns() -> None:
series.py
data = rv.get_json()
assert rv.status_code == 200
rows = data["rows"]
from dateutil import parser
timestamps = [parser.parse(r[0]).replace(tzinfo=None) for r in rows]
assert timestamps == sorted(timestamps)
def test_timeseries_count_no_columns() -> None:
app = server.app
client = app.test_client()
payload: dict[str, Any] = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "timeseries",
"granularity": "1 day",
"columns": [],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
rows = data["rows"]
assert len(rows) == 2
assert rows[0][1] == 2
assert rows[1][1] == 2
def test_timeseries_limit_applies_to_series() -> None:
app = server.app
The user requested a test for an "id" and "Hits" column; I'll create a Web test for the DOM and possibly a server test. Let's also inspect the database to check the data before proceeding.
The default dataset is sample.csv, but I'll adapt the server for the "TEST" dataset as per user request.
I'll open tests/test_multi_table_web.py to see how it customizes server_url for multi-table datasets.
from collections.abc import Iterator
from typing import Any
import pytest
from werkzeug.serving import make_server
from scubaduck.server import create_app
from tests.web_utils import select_value
@pytest.fixture()
def multi_table_server_url() -> Iterator[str]:
app = create_app("TEST")
httpd = make_server("127.0.0.1", 0, app)
port = httpd.server_port
thread = threading.Thread(target=httpd.serve_forever)
thread.start()
try:
yield f"http://127.0.0.1:{port}"
finally:
httpd.shutdown()
thread.join()
def test_table_param_updates_on_dive(page: Any, multi_table_server_url: str) ->
None:
page.goto(multi_table_server_url + "?table=events")
page.wait_for_selector("#table option", state="attached")
select_value(page, "#table", "extra")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
table_param = page.evaluate(
"new URLSearchParams(window.location.search).get('table')"
)
assert table_param == "extra"
def test_table_dropdown_persists_on_refresh(
page: Any, multi_table_server_url: str
) -> None:
page.goto(multi_table_server_url + "?table=events")
page.wait_for_selector("#table option", state="attached")
select_value(page, "#table", "extra")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
page.reload()
page.wait_for_selector("#table option", state="attached")
assert page.input_value("#table") == "extra"
disp = page.text_content("#table + .dropdown-display")
assert disp is not None and disp.strip() == "extra"
root@b8a745a412a8:/workspace/scubaduck# sed -n '80,160p' tests/test_multi_table_
web.py
I'll check if there's a generic fixture for the TEST dataset or if I need to adapt.
root@b8a745a412a8:/workspace/scubaduck# grep -n "create_app(\"TEST\")" -r tests
| head
tests/test_server_timeseries.py:253: app = server.create_app("TEST")
tests/test_server_time.py:234: app = server.create_app("TEST")
tests/test_server_time.py:252: app = server.create_app("TEST")
tests/test_multi_table_web.py:14: app = create_app("TEST")
I’ll add a fixture for the 'TEST' dataset, inspired by test_multi_table_web
.
s | head
tests/test_multi_table_web.py:13:def multi_table_server_url() -> Iterator[str]:
tests/test_multi_table_web.py:26:def test_table_param_updates_on_dive(page: Any,
multi_table_server_url: str) -> None:
tests/test_multi_table_web.py:27: page.goto(multi_table_server_url + "?table=
events")
tests/test_multi_table_web.py:40: page: Any, multi_table_server_url: str
tests/test_multi_table_web.py:42: page.goto(multi_table_server_url + "?table=
events")
I’ll create the test environment using create_app("TEST")
and run run_query
without specifying start/end—since the dataset is small enough, it should work fine. Let’s inspect the dataset for column dependencies.
The test might fail due to missing "value" column. I'll check how it's used in tests.
root@b8a745a412a8:/workspace/scubaduck# grep -n "run_query(" -n tests/test_multi
_table_web.py
I’ll adapt run_query to avoid relying on 'value' and check for checkboxes' existence.
root@b8a745a412a8:/workspace/scubaduck# sed -n '20,70p' tests/web_utils.py
order_dir: str | None = "ASC",
limit: int | None = None,
group_by: list[str] | None = None,
aggregate: str | None = None,
) -> dict[str, Any]:
page.goto(url)
page.wait_for_selector("#order_by option", state="attached")
page.wait_for_selector("#order_dir", state="attached")
page.wait_for_function("window.lastResults !== undefined")
if start is not None:
page.fill("#start", start)
if end is not None:
page.fill("#end", end)
if order_by is not None:
select_value(page, "#order_by", order_by)
if order_dir is not None and order_dir == "DESC":
page.click("#order_dir")
if limit is not None:
page.fill("#limit", str(limit))
if group_by is not None:
select_value(page, "#graph_type", "table")
page.evaluate(
"g => { groupBy.chips = g; groupBy.renderChips(); }",
group_by,
)
if aggregate is not None:
select_value(page, "#graph_type", "table")
select_value(page, "#aggregate", aggregate)
if page.input_value("#graph_type") != "samples":
page.click("text=Columns")
page.wait_for_selector("#column_groups input", state="attached")
if not page.is_checked("#column_groups input[value='value']"):
page.check("#column_groups input[value='value']")
order_col = order_by or page.input_value("#order_by")
if order_col and not page.is_checked(
f"#column_groups input[value='{order_col}']"
):
if page.query_selector(f"#column_groups input[value='{order_col}']")
:
page.check(f"#column_groups input[value='{order_col}']")
page.click("text=View Settings")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
return page.evaluate("window.lastResults")
Instead of using run_query, I'll directly set up the dataset and handle operations manually.
root@b8a745a412a8:/workspace/scubaduck# grep -n "columns_all" -n scubaduck/stati
c/index.html
337: <a id="columns_all" href="#">All</a>
I'll confirm the status of group checkboxes in this section.
root@b8a745a412a8:/workspace/scubaduck# sed -n '320,360p' scubaduck/static/index
.html
<option>p99.9</option>
<option>p99.99</option>
</select>
</div>
<div id="show_hits_field" class="field" style="display:none;">
<label>Show Hits</label>
<input id="show_hits" type="checkbox" checked>
</div>
<div id="filters">
<h4>Filters<span class="help" title="You can create as many filters as
you want. You can either write a filter using a UI or manual SQL. In the UI, fi
lter consists of a column name, a relation (e.g., =, !=, <, >) and then a text f
ield. The text field is a token input. It accepts multiple tokens for = relation
, in which case we match using an OR for all options.">[?]</span></h4>
<div id="filter_list"></div>
<button id="add_filter" type="button" onclick="addFilter()">Add Filter
</button>
</div>
<div id="query_info" style="margin-top:10px;"></div>
</div>
<div id="columns" class="tab-content">
<div id="column_actions">
<a id="columns_all" href="#">All</a>
<a id="columns_none" href="#">None</a>
</div>
<div id="column_groups"></div>
<div id="derived_columns">
<h4>Derived Columns</h4>
<div id="derived_list"></div>
<button id="add_derived" type="button" onclick="addDerived()">Add Deri
ved</button>
</div>
</div>
</div>
<div id="sidebar-resizer"></div>
<div id="view">
<table id="results"></table>
</div>
</div>
<script src="/js/chip_input.js"></script>
<script src="/js/timeseries_chart.js"></script>
<script src="/js/time_column.js"></script>
<script src="/js/sidebar.js"></script>
<script src="/js/table_view.js"></script>
<script src="/js/view_settings.js"></script>
</body>
</html>
I’ll add checks for row counts and table dimensions—ensuring there are 2 rows and no overflow.
I'll modify updateSelectedColumns
in table_view.js to exclude aggregated columns based on existing logic.
root@b8a745a412a8:/workspace/scubaduck# sed -n '430,470p' scubaduck/static/js/vi
ew_settings.js
}
function updateSelectedColumns(type = graphTypeSel.value) {
const base = allColumns.filter(name => {
const cb = document.querySelector(`#column_groups input[value="${name}"]`);
if (!cb || !cb.checked) return false;
if (type === 'table' && isStringColumn(name)) return false;
return true;
});
if (type === 'table' || type === 'timeseries') {
selectedColumns = groupBy.chips.slice();
if (document.getElementById('show_hits').checked) selectedColumns.push('Hits
');
base.forEach(c => {
if (!selectedColumns.includes(c)) selectedColumns.push(c);
});
derivedColumns.forEach(dc => {
if (dc.include && !selectedColumns.includes(dc.name)) selectedColumns.push
(dc.name);
});
} else {
selectedColumns = base.slice();
derivedColumns.forEach(dc => {
if (dc.include) selectedColumns.push(dc.name);
});
}
columnValues[type] = selectedColumns.slice();
const orderCol = document.getElementById('order_by').value;
if (orderCol && !selectedColumns.includes(orderCol)) {
selectedColumns.push(orderCol);
}
updateColumnsTabCount();
}
function isStringColumn(name) {
const t = (columnTypes[name] || '').toUpperCase();
return t.includes('CHAR') || t.includes('STRING') || t.includes('VARCHAR');
}
function isIntegerColumn(name) {
const t = (columnTypes[name] || '').toUpperCase();
return t.includes('INT');
}
I'll open the patch now to make the necessary changes.
root@b8a745a412a8:/workspace/scubaduck# sed -n '428,466p' scubaduck/static/js/vi
ew_settings.js
const btn = document.getElementById('columns_tab');
if (btn) btn.textContent = `Columns (${baseCount + derivedCount})`;
}
function updateSelectedColumns(type = graphTypeSel.value) {
const base = allColumns.filter(name => {
const cb = document.querySelector(`#column_groups input[value="${name}"]`);
if (!cb || !cb.checked) return false;
if (type === 'table' && isStringColumn(name)) return false;
return true;
});
if (type === 'table' || type === 'timeseries') {
selectedColumns = groupBy.chips.slice();
if (document.getElementById('show_hits').checked) selectedColumns.push('Hits
');
base.forEach(c => {
if (!selectedColumns.includes(c)) selectedColumns.push(c);
});
derivedColumns.forEach(dc => {
if (dc.include && !selectedColumns.includes(dc.name)) selectedColumns.push
(dc.name);
});
} else {
selectedColumns = base.slice();
derivedColumns.forEach(dc => {
if (dc.include) selectedColumns.push(dc.name);
});
}
columnValues[type] = selectedColumns.slice();
const orderCol = document.getElementById('order_by').value;
if (orderCol && !selectedColumns.includes(orderCol)) {
selectedColumns.push(orderCol);
}
updateColumnsTabCount();
}
function isStringColumn(name) {
const t = (columnTypes[name] || '').toUpperCase();
return t.includes('CHAR') || t.includes('STRING') || t.includes('VARCHAR');
}
I'll apply the patch using the apply_patch
method.
root@b8a745a412a8:/workspace/scubaduck# apply_patch<<'PATCH'
> *** Begin Patch
> *** Update File: scubaduck/static/js/view_settings.js
> @@
> - if (type === 'table' || type === 'timeseries') {
> - selectedColumns = groupBy.chips.slice();
> - if (document.getElementById('show_hits').checked) selectedColumns.push('H
its');
> - base.forEach(c => {
> - if (!selectedColumns.includes(c)) selectedColumns.push(c);
> - });
> - derivedColumns.forEach(dc => {
> - if (dc.include && !selectedColumns.includes(dc.name)) selectedColumns.p
ush(dc.name);
> - });
> - } else {
> + if (type === 'table' || type === 'timeseries') {
> + selectedColumns = groupBy.chips.slice();
> + if (document.getElementById('show_hits').checked) selectedColumns.push('H
its');
> + const agg = document.getElementById('aggregate').value.toLowerCase();
> + if (!(type === 'table' && agg === 'count')) {
> + base.forEach(c => {
> + if (!selectedColumns.includes(c)) selectedColumns.push(c);
> + });
> + derivedColumns.forEach(dc => {
> + if (dc.include && !selectedColumns.includes(dc.name)) selectedColumns
.push(dc.name);
> + });
> + }
> + } else {
> *** End Patch
> PATCH
Success. Updated the following files:
M scubaduck/static/js/view_settings.js
I'm adding a new fixture and updating tests after checking aggregate count.
root@b8a745a412a8:/workspace/scubaduck# sed -n '320,380p' tests/test_web_table.p
y
I'll verify the file's start to be sure there's no missing newline or issue.
from __future__ import annotations
from typing import Any
from tests.web_utils import run_query
def test_table_sorting(page: Any, server_url: str) -> None:
run_query(
page,
server_url,
start="2024-01-01 00:00:00",
end="2024-01-03 00:00:00",
order_by="user",
order_dir="ASC",
limit=100,
)
# header alignment
align = page.evaluate(
"getComputedStyle(document.querySelector('#results th')).textAlign"
)
assert align == "left"
header = page.locator("#results th").nth(3)
def values() -> list[str]:
return page.locator("#results td:nth-child(4)").all_inner_texts()
orig_rows = values()
assert orig_rows == ["alice", "bob", "alice", "charlie"]
first_sql = page.evaluate("window.lastResults.sql")
header.click()
assert values() == sorted(orig_rows)
assert header.inner_text().endswith("▲")
color = page.evaluate(
"getComputedStyle(document.querySelector('#results th:nth-child(4)')).co
lor"
)
assert "0, 0, 255" in color
I'll add the new fixture and test at the file's bottom, checking for a newline first.
root@b8a745a412a8:/workspace/scubaduck# tail -n 20 tests/test_web_table.py
page.click("text=Add Derived")
assert page.query_selector("#derived_list .derived button.remove")
page.click("#derived_list .derived button.remove")
count = page.evaluate("document.querySelectorAll('#derived_list .derived').l
ength")
assert count == 0
def test_sql_query_display(page: Any, server_url: str) -> None:
data = run_query(
page,
server_url,
start="2024-01-01 00:00:00",
end="2024-01-02 00:00:00",
order_by="timestamp",
limit=10,
)
sql = data["sql"]
displayed = page.text_content("#sql_query")
assert displayed is not None
assert displayed.strip() == sql
I’ll add the new fixture and patch the imports for select_value
and others.
> *** Begin Patch
> *** Update File: tests/test_web_table.py
> @@
> -from tests.web_utils import run_query
> +from collections.abc import Iterator
> +import threading
> +
> +import pytest
> +from werkzeug.serving import make_server
> +
> +from scubaduck.server import create_app
> +from tests.web_utils import run_query, select_value
> *** End Patch
> PATCH
Success. Updated the following files:
M tests/test_web_table.py
I'll add the fixture near the top or bottom of the test file.
root@b8a745a412a8:/workspace/scubaduck# grep -n "multi_table_server_url" -n test
s/test_web_table.py
I'll add the missing multi_table fixture right after the imports.
root@b8a745a412a8:/workspace/scubaduck# head -n 20 tests/test_web_table.py
from __future__ import annotations
from typing import Any
from collections.abc import Iterator
import threading
import pytest
from werkzeug.serving import make_server
from scubaduck.server import create_app
from tests.web_utils import run_query, select_value
def test_table_sorting(page: Any, server_url: str) -> None:
run_query(
page,
server_url,
start="2024-01-01 00:00:00",
end="2024-01-03 00:00:00",
I'll add the fixture right after the imports but before the first test.
> *** Begin Patch
> *** Update File: tests/test_web_table.py
> @@
> -from scubaduck.server import create_app
> -from tests.web_utils import run_query, select_value
> +from scubaduck.server import create_app
> +from tests.web_utils import run_query, select_value
> +
> +
> +@pytest.fixture()
> +def test_dataset_server_url() -> Iterator[str]:
> + app = create_app("TEST")
> + httpd = make_server("127.0.0.1", 0, app)
> + port = httpd.server_port
> + thread = threading.Thread(target=httpd.serve_forever)
> + thread.start()
> + try:
> + yield f"http://127.0.0.1:{port}"
> + finally:
> + httpd.shutdown()
> + thread.join()
> *** End Patch
> PATCH
Success. Updated the following files:
M tests/test_web_table.py
Adding the new test toward the end should work best here.
page.click("text=Add Derived")
assert page.query_selector("#derived_list .derived button.remove")
page.click("#derived_list .derived button.remove")
count = page.evaluate("document.querySelectorAll('#derived_list .derived').l
ength")
assert count == 0
def test_sql_query_display(page: Any, server_url: str) -> None:
data = run_query(
page,
server_url,
start="2024-01-01 00:00:00",
end="2024-01-02 00:00:00",
order_by="timestamp",
limit=10,
)
sql = data["sql"]
displayed = page.text_content("#sql_query")
assert displayed is not None
assert displayed.strip() == sql
I'll follow up with a new test after this one.
> *** Update File: tests/test_web_table.py
> @@
> def test_sql_query_display(page: Any, server_url: str) -> None:
> @@
> assert displayed.strip() == sql
> +
> +
> +def test_table_count_no_columns(page: Any, test_dataset_server_url: str) -> N
one:
> + page.goto(test_dataset_server_url)
> + page.wait_for_selector("#order_by option", state="attached")
> + select_value(page, "#graph_type", "table")
> + page.click("text=Columns")
> + page.click("#columns_all")
> + page.click("text=View Settings")
> + page.evaluate("groupBy.chips = ['id']; groupBy.renderChips();")
> + select_value(page, "#aggregate", "Count")
> + page.evaluate("window.lastResults = undefined")
> + page.click("text=Dive")
> + page.wait_for_function("window.lastResults !== undefined")
> + headers = page.locator("#results th").all_inner_texts()
> + assert headers == ["id", "Hits"]
> + col_count = page.locator("#results th").count()
> + row_count = page.locator("#results tr").count()
> + assert col_count == 2
> + assert row_count == 3
> + overflow = page.evaluate(
> + "var v=document.getElementById('view'); v.scrollWidth > v.clientWidth
"
> + )
> + assert not overflow
> *** End Patch
> PATCH
Success. Updated the following files:
M tests/test_web_table.py
I'll run ruff format, ruff check, and pyright to verify the code.
root@b8a745a412a8:/workspace/scubaduck# source .venv/bin/activate
(scubaduck) root@b8a745a412a8:/workspace/scubaduck# ruff format scubaduck/static
/js/view_settings.js tests/test_web_table.py >/tmp/format.log && tail -n 20 /tmp
/format.log
error: Failed to parse scubaduck/static/js/view_settings.js:1:1: Expected a stat
ement
(scubaduck) root@b8a745a412a8:/workspace/scubaduck#
Ruff is for Python, so I'll only run ruff format
on Python files.
(scubaduck) root@b8a745a412a8:/workspace/scubaduck# ruff format tests/test_web_t
able.py >/tmp/format.log && tail -n 20 /tmp/format.log
1 file left unchanged
(scubaduck) root@b8a745a412a8:/workspace/scubaduck# ruff check scubaduck tests >
/tmp/ruff.log && tail -n 20 /tmp/ruff.log
All checks passed!
(scubaduck) root@b8a745a412a8:/workspace/scubaduck#
I'll go ahead and run pyright to check for any type errors.
(scubaduck) root@b8a745a412a8:/workspace/scubaduck# pyright >/tmp/pyright.log &&
tail -n 20 /tmp/pyright.log
0 errors, 0 warnings, 0 informations
(scubaduck) root@b8a745a412a8:/workspace/scubaduck# cat /tmp/pyright.log
0 errors, 0 warnings, 0 informations
(scubaduck) root@b8a745a412a8:/workspace/scubaduck#
I'll run pytest to check everything is working smoothly.
(scubaduck) root@b8a745a412a8:/workspace/scubaduck# pytest -q >/tmp/pytest.log &
& tail -n 20 /tmp/pytest.log
bringing up nodes...
bringing up nodes...
........................................................................ [ 62%]
............................................ [100%]
116 passed in 17.72s
(scubaduck) root@b8a745a412a8:/workspace/scubaduck#
Everything passed! I'll check the log now for any additional details.
(scubaduck) root@b8a745a412a8:/workspace/scubaduck# cat /tmp/pytest.log | tail -
n 20
bringing up nodes...
bringing up nodes...
........................................................................ [ 62%]
............................................ [100%]
116 passed in 17.72s
(scubaduck) root@b8a745a412a8:/workspace/scubaduck#
I'm about to commit my changes now!
(scubaduck) root@b8a745a412a8:/workspace/scubaduck# git status --short
M scubaduck/static/js/view_settings.js
M tests/test_web_table.py
(scubaduck) root@b8a745a412a8:/workspace/scubaduck# git add scubaduck/static/js/
view_settings.js tests/test_web_table.py
(scubaduck) root@b8a745a412a8:/workspace/scubaduck# git commit -m "feat(web): hi
de agg columns when count" && git status --short
[work 1533f3e] feat(web): hide agg columns when count
2 files changed, 55 insertions(+), 7 deletions(-)
(scubaduck) root@b8a745a412a8:/workspace/scubaduck#
I'll check the status of the repository to see if anything’s changed.
(scubaduck) root@b8a745a412a8:/workspace/scubaduck# git status --short
(scubaduck) root@b8a745a412a8:/workspace/scubaduck#
I’ll gather the line ranges for the changes in view_settings.js.
430 }
431
432 function updateSelectedColumns(type = graphTypeSel.value) {
433 const base = allColumns.filter(name => {
434 const cb = document.querySelector(`#column_groups input[value="${nam
e}"]`);
435 if (!cb || !cb.checked) return false;
436 if (type === 'table' && isStringColumn(name)) return false;
437 return true;
438 });
439 if (type === 'table' || type === 'timeseries') {
440 selectedColumns = groupBy.chips.slice();
441 if (document.getElementById('show_hits').checked) selectedColumns.pu
sh('Hits');
442 const agg = document.getElementById('aggregate').value.toLowerCase()
;
443 if (!(type === 'table' && agg === 'count')) {
444 base.forEach(c => {
445 if (!selectedColumns.includes(c)) selectedColumns.push(c);
446 });
447 derivedColumns.forEach(dc => {
448 if (dc.include && !selectedColumns.includes(dc.name)) selectedCo
lumns.push(dc.name);
449 });
450 }
451 } else {
452 selectedColumns = base.slice();
453 derivedColumns.forEach(dc => {
454 if (dc.include) selectedColumns.push(dc.name);
455 });
456 }
457 columnValues[type] = selectedColumns.slice();
458 const orderCol = document.getElementById('order_by').value;
459 if (orderCol && !selectedColumns.includes(orderCol)) {
460 selectedColumns.push(orderCol);
461 }
462 updateColumnsTabCount();
(scubaduck) root@b8a745a412a8:/workspace/scubaduck#
I'll gather the relevant lines from tests/test_web_table.py for the new test reference.
(scubaduck) root@b8a745a412a8:/workspace/scubaduck# nl -ba tests/test_web_table.
py | sed -n '1,40p'
1 from __future__ import annotations
2
3 from typing import Any
4
5 from collections.abc import Iterator
6 import threading
7
8 import pytest
9 from werkzeug.serving import make_server
10
11 from scubaduck.server import create_app
12 from tests.web_utils import run_query, select_value
13
14
15 @pytest.fixture()
16 def test_dataset_server_url() -> Iterator[str]:
17 app = create_app("TEST")
18 httpd = make_server("127.0.0.1", 0, app)
19 port = httpd.server_port
20 thread = threading.Thread(target=httpd.serve_forever)
21 thread.start()
22 try:
23 yield f"http://127.0.0.1:{port}"
24 finally:
25 httpd.shutdown()
26 thread.join()
27
28
29 def test_table_sorting(page: Any, server_url: str) -> None:
30 run_query(
31 page,
32 server_url,
33 start="2024-01-01 00:00:00",
34 end="2024-01-03 00:00:00",
35 order_by="user",
36 order_dir="ASC",
37 limit=100,
38 )
39 # header alignment
40 align = page.evaluate(
(scubaduck) root@b8a745a412a8:/workspace/scubaduck#
I’ll grab the line numbers where the new test was added.
(scubaduck) root@b8a745a412a8:/workspace/scubaduck# nl -ba tests/test_web_table.
py | sed -n '188,240p'
188
189 def test_group_by_autocomplete(page: Any, server_url: str) -> None:
190 page.goto(f"{server_url}?graph_type=table")
191 page.wait_for_selector("#group_by_field", state="visible")
192 inp = page.query_selector("#group_by_field .f-val")
193 assert inp
194 inp.click()
195 page.keyboard.type("us")
196 page.wait_for_selector("#group_by_field .chip-dropdown div")
197 options = page.locator("#group_by_field .chip-dropdown div").all_inn
er_texts()
198 assert "user" in options
199
200
201 def test_group_by_copy_icon(page: Any, server_url: str) -> None:
202 page.goto(f"{server_url}?graph_type=table")
203 page.wait_for_selector("#group_by_field", state="visible")
204 icon = page.text_content("#group_by_field .chip-copy")
205 assert icon == "⎘"
206
207
208 def test_group_by_input_no_border(page: Any, server_url: str) -> None:
209 page.goto(f"{server_url}?graph_type=table")
210 page.wait_for_selector("#group_by_field", state="visible")
211 border = page.evaluate(
212 "getComputedStyle(document.querySelector('#group_by_field .f-val
')).borderStyle"
213 )
214 assert border == "none"
215
216
217 def test_table_group_by_query(page: Any, server_url: str) -> None:
218 data = run_query(
219 page,
220 server_url,
221 start="2024-01-01 00:00:00",
222 end="2024-01-03 00:00:00",
223 order_by="user",
224 limit=100,
225 group_by=["user"],
226 aggregate="Count",
227 )
228 assert "error" not in data
229 assert len(data["rows"]) == 3
230
231
232 def test_table_avg_no_group_by(page: Any, server_url: str) -> None:
233 data = run_query(
234 page,
235 server_url,
236 aggregate="Avg",
237 )
238 assert len(data["rows"]) == 1
239 row = data["rows"][0]
240 assert row[0] == 4
(scubaduck) root@b8a745a412a8:/workspace/scubaduck#
I'm checking the new test at the bottom and making sure everything's in place.
(scubaduck) root@b8a745a412a8:/workspace/scubaduck# nl -ba tests/test_web_table.
py | sed -n '240,340p'
240 assert row[0] == 4
241 from dateutil import parser
242
243 ts = parser.parse(row[1]).replace(tzinfo=None)
244 assert ts == parser.parse("2024-01-01 13:00:00")
245 assert row[2] == 25
246
247
248 def test_table_headers_show_aggregate(page: Any, server_url: str) -> Non
e:
249 run_query(
250 page,
251 server_url,
252 aggregate="Avg",
253 )
254 headers = page.locator("#results th").all_inner_texts()
255 assert "Hits" in headers
256 assert "timestamp (avg)" in headers
257 assert "value (avg)" in headers
258
259
260 def test_format_number_function(page: Any, server_url: str) -> None:
261 page.goto(server_url)
262 page.wait_for_selector("#order_by option", state="attached")
263 vals = page.evaluate(
264 "() => [formatNumber(815210), formatNumber(999.999), formatNumbe
r(0.0004), formatNumber(0)]"
265 )
266 assert vals == ["815.21 K", "999.999", "0.000", "0"]
267
268
269 def test_numeric_cell_nowrap(page: Any, server_url: str) -> None:
270 run_query(page, server_url, limit=10)
271 whitespace = page.evaluate(
272 "getComputedStyle(document.querySelector('#results td:nth-child(
3)')).whiteSpace"
273 )
274 assert whitespace == "nowrap"
275
276
277 def test_date_cell_nowrap(page: Any, server_url: str) -> None:
278 run_query(page, server_url, limit=10)
279 whitespace = page.evaluate(
280 "getComputedStyle(document.querySelector('#results td:nth-child(
1)')).whiteSpace"
281 )
282 assert whitespace == "nowrap"
283
284
285 def test_derived_column_query(page: Any, server_url: str) -> None:
286 page.goto(server_url)
287 page.wait_for_selector("#order_by option", state="attached")
288 page.click("text=Columns")
289 page.click("text=Add Derived")
290 expr = page.query_selector("#derived_list .derived textarea")
291 assert expr
292 expr.fill("value * 2")
293 page.click("text=View Settings")
294 page.fill("#start", "2024-01-01 00:00:00")
295 page.fill("#end", "2024-01-03 00:00:00")
296 page.fill("#limit", "10")
297 page.evaluate("window.lastResults = undefined")
298 page.click("text=Dive")
299 page.wait_for_function("window.lastResults !== undefined")
300 data = page.evaluate("window.lastResults")
301 assert data["rows"][0][-1] == 20
302
303
304 def test_derived_column_remove(page: Any, server_url: str) -> None:
305 page.goto(server_url)
306 page.wait_for_selector("#order_by option", state="attached")
307 page.click("text=Columns")
308 page.click("text=Add Derived")
309 assert page.query_selector("#derived_list .derived button.remove")
310 page.click("#derived_list .derived button.remove")
311 count = page.evaluate("document.querySelectorAll('#derived_list .der
ived').length")
312 assert count == 0
313
314
315 def test_sql_query_display(page: Any, server_url: str) -> None:
316 data = run_query(
317 page,
318 server_url,
319 start="2024-01-01 00:00:00",
320 end="2024-01-02 00:00:00",
321 order_by="timestamp",
322 limit=10,
323 )
324 sql = data["sql"]
325 displayed = page.text_content("#sql_query")
326 assert displayed is not None
327 assert displayed.strip() == sql
328
329
330 def test_table_count_no_columns(page: Any, test_dataset_server_url: str)
-> None:
331 page.goto(test_dataset_server_url)
332 page.wait_for_selector("#order_by option", state="attached")
333 select_value(page, "#graph_type", "table")
334 page.click("text=Columns")
335 page.click("#columns_all")
336 page.click("text=View Settings")
337 page.evaluate("groupBy.chips = ['id']; groupBy.renderChips();")
338 select_value(page, "#aggregate", "Count")
339 page.evaluate("window.lastResults = undefined")
340 page.click("text=Dive")
(scubaduck) root@b8a745a412a8:/workspace/scubaduck# nl -ba tests/test_web_table.
py | sed -n '340,372p'
340 page.click("text=Dive")
341 page.wait_for_function("window.lastResults !== undefined")
342 headers = page.locator("#results th").all_inner_texts()
343 assert headers == ["id", "Hits"]
344 col_count = page.locator("#results th").count()
345 row_count = page.locator("#results tr").count()
346 assert col_count == 2
347 assert row_count == 3
348 overflow = page.evaluate(
349 "var v=document.getElementById('view'); v.scrollWidth > v.client
Width"
350 )
351 assert not overflow
(scubaduck) root@b8a745a412a8:/workspace/scubaduck#
I’ll summarize the change, referencing the JS patch from lines 433 to 451 and the new test.