Task ID: task_e_682fdbae60788323a9082313aff3f6f9
Configuring language runtimes...
+ rm -- /tmp/bILhWz-setup_script.sh
+ source ./codex_setup.sh
++ uv sync --frozen
Using CPython 3.12.3 interpreter at: /usr/bin/python3.12
Creating virtual environment at: .venv
Building scubaduck @ file:///workspace/scubaduck
Downloading duckdb (19.3MiB)
Downloading pyright (5.3MiB)
Downloading ruff (11.0MiB)
Downloading playwright (43.1MiB)
Downloading ruff
Built scubaduck @ file:///workspace/scubaduck
Downloading duckdb
Downloading playwright
Downloading pyright
Prepared 33 packages in 1.70s
Installed 33 packages in 81ms
+ blinker==1.9.0
+ certifi==2025.4.26
+ charset-normalizer==3.4.2
+ click==8.2.0
+ duckdb==1.2.2
+ execnet==2.1.1
+ flask==3.1.1
+ greenlet==3.2.2
+ idna==3.10
+ iniconfig==2.1.0
+ itsdangerous==2.2.0
+ jinja2==3.1.6
+ markupsafe==3.0.2
+ nodeenv==1.9.1
+ packaging==25.0
+ playwright==1.52.0
+ pluggy==1.6.0
+ pyee==13.0.0
+ pyright==1.1.400
+ pytest==8.3.5
+ pytest-base-url==2.1.0
+ pytest-playwright==0.7.0
+ pytest-xdist==3.6.1
+ python-dateutil==2.9.0.post0
+ python-slugify==8.0.4
+ requests==2.32.3
+ ruff==0.11.10
+ scubaduck==0.1.0 (from file:///workspace/scubaduck)
+ six==1.17.0
+ text-unidecode==1.3
+ typing-extensions==4.13.2
+ urllib3==2.4.0
+ werkzeug==3.1.3
++ source .venv/bin/activate
+++ '[' -z '' ']'
+++ '[' -n x ']'
+++ SCRIPT_PATH=.venv/bin/activate
+++ '[' .venv/bin/activate = /tmp/bILhWz-setup_script.sh ']'
+++ deactivate nondestructive
+++ unset -f pydoc
+++ '[' -z '' ']'
+++ '[' -z '' ']'
+++ hash -r
+++ '[' -z '' ']'
+++ unset VIRTUAL_ENV
+++ unset VIRTUAL_ENV_PROMPT
+++ '[' '!' nondestructive = nondestructive ']'
+++ VIRTUAL_ENV=/workspace/scubaduck/.venv
+++ '[' linux-gnu = cygwin ']'
+++ '[' linux-gnu = msys ']'
+++ export VIRTUAL_ENV
+++ '[' -z '' ']'
+++ unset SCRIPT_PATH
+++ _OLD_VIRTUAL_PATH=/root/.cargo/bin:/root/.rbenv/shims:/root/.rbenv/bin:/root/.rbenv/shims:/root/.local/share/swiftly/bin:/root/.bun/bin:/root/.nvm/versions/node/v22.16.0/bin:/root/.pyenv/shims:3441PYENV_ROOT/shims:/root/.pyenv/bin:/usr/local/go/bin:/root/go/bin:/root/.rbenv/bin:/root/.rbenv/shims:/root/.bun/bin:/root/.local/bin:/root/.pyenv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+++ PATH=/workspace/scubaduck/.venv/bin:/root/.cargo/bin:/root/.rbenv/shims:/root/.rbenv/bin:/root/.rbenv/shims:/root/.local/share/swiftly/bin:/root/.bun/bin:/root/.nvm/versions/node/v22.16.0/bin:/root/.pyenv/shims:3441PYENV_ROOT/shims:/root/.pyenv/bin:/usr/local/go/bin:/root/go/bin:/root/.rbenv/bin:/root/.rbenv/shims:/root/.bun/bin:/root/.local/bin:/root/.pyenv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+++ export PATH
+++ '[' xscubaduck '!=' x ']'
+++ VIRTUAL_ENV_PROMPT=scubaduck
+++ export VIRTUAL_ENV_PROMPT
+++ '[' -z '' ']'
+++ '[' -z '' ']'
+++ _OLD_VIRTUAL_PS1=
+++ PS1='(scubaduck) '
+++ export PS1
+++ alias pydoc
+++ true
+++ hash -r
++ python -c 'import os; import duckdb; con = duckdb.connect(); con.execute(f"SET http_proxy = '\''{os.getenv("HTTP_PROXY")}'\''"); con.execute("INSTALL '\''sqlite'\'';")'
++ playwright install chromium
Downloading Chromium 136.0.7103.25 (playwright build v1169) from https://cdn.playwright.dev/dbazure/download/playwright/builds/chromium/1169/chromium-linux.zip
167.7 MiB [] 0% 0.0s167.7 MiB [] 0% 32.2s167.7 MiB [] 0% 24.8s167.7 MiB [] 0% 15.8s167.7 MiB [] 0% 9.7s167.7 MiB [] 1% 6.2s167.7 MiB [] 2% 4.4s167.7 MiB [] 3% 2.9s167.7 MiB [] 5% 2.4s167.7 MiB [] 6% 2.3s167.7 MiB [] 7% 2.1s167.7 MiB [] 8% 1.9s167.7 MiB [] 9% 1.8s167.7 MiB [] 10% 1.8s167.7 MiB [] 12% 1.6s167.7 MiB [] 14% 1.5s167.7 MiB [] 15% 1.4s167.7 MiB [] 17% 1.3s167.7 MiB [] 19% 1.2s167.7 MiB [] 22% 1.1s167.7 MiB [] 24% 1.0s167.7 MiB [] 26% 1.0s167.7 MiB [] 27% 0.9s167.7 MiB [] 29% 0.9s167.7 MiB [] 32% 0.8s167.7 MiB [] 34% 0.8s167.7 MiB [] 36% 0.7s167.7 MiB [] 38% 0.7s167.7 MiB [] 39% 0.7s167.7 MiB [] 40% 0.7s167.7 MiB [] 41% 0.7s167.7 MiB [] 42% 0.7s167.7 MiB [] 42% 0.8s167.7 MiB [] 43% 0.8s167.7 MiB [] 45% 0.7s167.7 MiB [] 47% 0.7s167.7 MiB [] 49% 0.6s167.7 MiB [] 52% 0.6s167.7 MiB [] 54% 0.6s167.7 MiB [] 56% 0.5s167.7 MiB [] 58% 0.5s167.7 MiB [] 60% 0.5s167.7 MiB [] 62% 0.4s167.7 MiB [] 65% 0.4s167.7 MiB [] 66% 0.4s167.7 MiB [] 69% 0.4s167.7 MiB [] 70% 0.3s167.7 MiB [] 72% 0.3s167.7 MiB [] 74% 0.3s167.7 MiB [] 75% 0.3s167.7 MiB [] 78% 0.2s167.7 MiB [] 80% 0.2s167.7 MiB [] 82% 0.2s167.7 MiB [] 84% 0.2s167.7 MiB [] 86% 0.1s167.7 MiB [] 88% 0.1s167.7 MiB [] 90% 0.1s167.7 MiB [] 92% 0.1s167.7 MiB [] 94% 0.1s167.7 MiB [] 96% 0.0s167.7 MiB [] 98% 0.0s167.7 MiB [] 100% 0.0s
Chromium 136.0.7103.25 (playwright build v1169) downloaded to /root/.cache/ms-playwright/chromium-1169
Downloading FFMPEG playwright build v1011 from https://cdn.playwright.dev/dbazure/download/playwright/builds/ffmpeg/1011/ffmpeg-linux.zip
2.3 MiB [] 0% 0.0s2.3 MiB [] 4% 0.5s2.3 MiB [] 10% 0.3s2.3 MiB [] 22% 0.2s2.3 MiB [] 49% 0.1s2.3 MiB [] 97% 0.0s2.3 MiB [] 100% 0.0s
FFMPEG playwright build v1011 downloaded to /root/.cache/ms-playwright/ffmpeg-1011
Downloading Chromium Headless Shell 136.0.7103.25 (playwright build v1169) from https://cdn.playwright.dev/dbazure/download/playwright/builds/chromium/1169/chromium-headless-shell-linux.zip
101.4 MiB [] 0% 0.0s101.4 MiB [] 0% 18.5s101.4 MiB [] 0% 17.9s101.4 MiB [] 0% 12.3s101.4 MiB [] 0% 7.5s101.4 MiB [] 1% 4.5s101.4 MiB [] 3% 2.6s101.4 MiB [] 5% 1.9s101.4 MiB [] 7% 1.7s101.4 MiB [] 9% 1.5s101.4 MiB [] 11% 1.3s101.4 MiB [] 12% 1.3s101.4 MiB [] 13% 1.2s101.4 MiB [] 13% 1.4s101.4 MiB [] 13% 1.5s101.4 MiB [] 14% 1.6s101.4 MiB [] 15% 1.5s101.4 MiB [] 18% 1.3s101.4 MiB [] 21% 1.2s101.4 MiB [] 22% 1.1s101.4 MiB [] 23% 1.2s101.4 MiB [] 23% 1.3s101.4 MiB [] 24% 1.4s101.4 MiB [] 25% 1.4s101.4 MiB [] 26% 1.4s101.4 MiB [] 27% 1.4s101.4 MiB [] 30% 1.2s101.4 MiB [] 31% 1.2s101.4 MiB [] 35% 1.0s101.4 MiB [] 37% 1.0s101.4 MiB [] 40% 0.9s101.4 MiB [] 41% 0.9s101.4 MiB [] 43% 0.8s101.4 MiB [] 45% 0.8s101.4 MiB [] 46% 0.8s101.4 MiB [] 49% 0.7s101.4 MiB [] 51% 0.8s101.4 MiB [] 52% 0.8s101.4 MiB [] 54% 0.7s101.4 MiB [] 56% 0.7s101.4 MiB [] 57% 0.7s101.4 MiB [] 58% 0.7s101.4 MiB [] 59% 0.7s101.4 MiB [] 61% 0.6s101.4 MiB [] 62% 0.6s101.4 MiB [] 63% 0.6s101.4 MiB [] 65% 0.5s101.4 MiB [] 66% 0.5s101.4 MiB [] 67% 0.5s101.4 MiB [] 69% 0.5s101.4 MiB [] 70% 0.5s101.4 MiB [] 71% 0.5s101.4 MiB [] 72% 0.5s101.4 MiB [] 74% 0.4s101.4 MiB [] 76% 0.4s101.4 MiB [] 78% 0.3s101.4 MiB [] 79% 0.3s101.4 MiB [] 80% 0.3s101.4 MiB [] 82% 0.3s101.4 MiB [] 83% 0.3s101.4 MiB [] 86% 0.2s101.4 MiB [] 87% 0.2s101.4 MiB [] 88% 0.2s101.4 MiB [] 89% 0.2s101.4 MiB [] 90% 0.2s101.4 MiB [] 91% 0.2s101.4 MiB [] 92% 0.2s101.4 MiB [] 93% 0.1s101.4 MiB [] 95% 0.1s101.4 MiB [] 96% 0.1s101.4 MiB [] 99% 0.0s101.4 MiB [] 100% 0.0s
Chromium Headless Shell 136.0.7103.25 (playwright build v1169) downloaded to /root/.cache/ms-playwright/chromium_headless_shell-1169
++ echo 'source .venv/bin/activate'I'll check the root of the repository to find AGENTS.md as directed.
bash: .venv/bin/activate: No such file or directory
root@3f0164aeb2f9:/workspace# ls
scubaduckroot@3f0164aeb2f9:/workspace# cd scubaduck
root@3f0164aeb2f9:/workspace/scubaduck# ls
AGENTS.md codex_setup.sh scubaduck tests
README.md pyproject.toml stubs uv.lockroot@3f0164aeb2f9:/workspace/scubaduck# cat AGENTS.md
- Use "source .venv/bin/activate" to activate the virtual environment before
running any commands
- ONLY use non-interactive commands like cat, sed, apply_patch to do edits.
Do NOT use interactive editors.
- Do NOT attempt to install packages. Only the packages specified in
pyproject.toml are available. You cannot add new packages. If you
desperately want another package, make a note of it in the final PR
description.
- Use conventional commits to format PR title
- There are no nested AGENTS.md files, this is the only agents file
- When using Playwright, ONLY use chromium browser. This browser is already
installed.
- Use "ruff check" to check lint, "ruff format" to autoformat files and
"pyright" to typecheck.
- When writing the PR description, include the original prompt supplied by the
user.
- When you add functionality to the server, add server tests to
tests/test_server.py. When you add features to the frontend, add frontend
tests to tests/test_web.pyI need to review the README for testing instructions and follow .venv setup.
root@3f0164aeb2f9:/workspace/scubaduck# cat README.md | head -n 40
I really like Scuba (Meta's internal real-time database system). The distributed
, real-time database part of Scuba is quite difficult (and expensive) to replica
te, but I also really like Scuba's UI for doing queries, and I have found myself
wishing that I have access to it even for "small" databases, e.g., I have a sql
ite dataset I want to explore.
Pivotal ideas:
* Time series by default. In the dedicated "time series" view, there are many fe
atures specifically oriented towards working towards tables that represent event
s that occurred over time: the start, end, compare, aggregate and granularity fi
elds all specially privilege the timestamp field. In fact, you can't log events
to Scuba's backing data store without a timestamp, they always come with one. (S
cuba also supports other views that don't presuppose a time series, but the time
series is the most beloved and well used view.) This is in contrast to typical
software which tries to generalize to arbitrary data first, with time series bei
ng added on later.
* It's all about exploration. Scuba is predicated on the idea that you don't kno
w what you're looking for, that you are going to spend time tweaking queries and
changing filters/grouping as part of an investigation to figure out why a syste
m behaves the way it is. So the filters/comparisons/groupings you want to edit a
re always visible on the left sidebar, with the expectation that you're going to
tweak the query to look at something else. Similarly, all the parameters of you
r query get saved into your URL, so your browser history can double up as a quer
y history / you can easily share a query with someone else. This is contrast to
typical software which is often oriented to making pretty dashboards and reports
. (This function is important too, but it's not what I want in exploration mode!
)
* You can fix data problems in the query editor. It's pretty common to have mess
ed up and ended up with a database that doesn't have exactly the columns you nee
d, or some columns that are corrupted in some way. Scuba has pretty robust suppo
rt for defining custom columns with arbitrary SQL functions, grouping over them
as if they were native functions, and doing so with minimal runtime cost (Scuba
aims to turn around your query in milliseconds!) Having to go and run a huge dat
a pipeline to fix your data is a big impediment to exploration; quick and easy c
ustom columns means you can patch over problems when you're investigating and fi
x them for real later.
We're going to build a exploratory data analysis tool like Scuba for time series
database (i.e., a database with a mandatory timestamp representing the time an
event occurred). We'll use DuckDB as the underlying SQL engine served from a Py
thon server, and render the GUI/results as a webpage with vanilla HTML and JS. W
e'll use choices.js to support token inputs. We define a token input to mean a
text input element where as you type a dropdown displays with valid values, and
if you select one or press enter, the selection turns into a token/chip that can
only be deleted as one unit.
To start, we are going to support one views: samples. The samples view only all
ows you to view individual samples from the database, subject to a filter. Our m
ain UI concept is that there is a left sidebar that is the query editor, and the
right side that shows the view. The sidebar is always visible and defaults to
the query parameters of the current view. After you make changes to the query,
clicking the "Dive" button updates the view. The URL of the page encodes all of
the values of the query (and gets updated when you Dive), so the browser's back
button lets you view previous queries.
The query editor's job is to generate a SQL query, which then is applied on the
database, and then the result visualized according to the view.
Here are the settings you can apply to the query. The help text should show up w
hen you mouse over the field name:
* Start/End - Help text: "Sets the start/end of the time range to query. Can be
any kind of datetime string. For example: 'April 23, 2014' or 'yesterday'." The
UI for this selector supports both relative selections (now, -1 hour, -3 hours,
-12 hours, -1 day, -3 days, -1 week, -1 fortnight, -30 days, -90 days, -1 month,
-1 year) as well as specifying an absolute date. The way this field is rendere
d is there is a free form text box, a drop down arrow (for the relative selector
s), and then a calendar button (for date selection).
* Order By - Help text: "Choose a column to sort results by." There is an ASC/D
ESC toggle next to it.
* Limit - Help text: "Choose the maximum number of results to show in the chart
after any aggregations have been applied. For example, a limit of 10 will show
no more than 10 rows for a table, etc."
* Filters - You can create as many filters as you want. You can either write a f
ilter using a UI or manual SQL. In the UI, filter consists of a column name, a r
elation (e.g., =, !=, <, >) and then a text field. The text field is a token inp
ut. It accepts multiple tokens for = relation, in which case we match using an O
R for all options.
There is also a "Columns" tab which lets you view all fields in the table, organ
ized by their type. You can also define derived columns, by specifying a column
name and SQL expression. Derived columns can be used for all parts of the UI, in
cluding filters/group by/etc. Columns have checkboxes indicating if we should SE
LECT them or not. Each selected column shows up in the graph. There is an All/N
one link which can be used to select/deselect all checkboxes.
The query UI constructs a SQL query that intuitively has this form:
```
SELECT column, column, ...,
FROM table
WHERE time >= min-timestamp
AND time <= max-timestamp
[AND condition ...]
ORDER BY aggregate(column)
LIMIT number
```
You should write tests for the server backend, demonstrating that at specific qu
ery values we get back the correct rows of data.
## Running the serverI’m looking for test-related instructions towards the end of the file.
root@3f0164aeb2f9:/workspace/scubaduck# sed -n '40,120p' README.md
## Running the server
Activate the virtual environment and run the Flask development server:
```bash
flask --app scubaduck.server run --debug
```
By default the server loads `sample.csv`. Set the `SCUBADUCK_DB` environment
variable to point at a different database file (CSV, SQLite or DuckDB) if you
want to use another dataset. The special value `TEST` starts the server with a
small in-memory SQLite dataset used by the automated tests. If the file does
not exist, the server will raise a `FileNotFoundError` during startup.I'm checking the bottom of the document for any test-related instructions.
root@3f0164aeb2f9:/workspace/scubaduck# grep -n "Running tests" -n README.mdMaybe the test instructions aren't in the files. I'll check to confirm.
root@3f0164aeb2f9:/workspace/scubaduck# grep -n "test" README.md
38:You should write tests for the server backend, demonstrating that at specific
query values we get back the correct rows of data.
51:small in-memory SQLite dataset used by the automated tests. If the file doesI'll browse the tests directory to find any relevant instructions.
root@3f0164aeb2f9:/workspace/scubaduck# ls tests
conftest.py test_server_time.py test_web_graph.py
test_multi_table_web.py test_server_timeseries.py test_web_table.py
test_server.py test_web.py test_web_timeseries.py
test_server_basic.py test_web_basics.py web_utils.py
test_server_db_types.py test_web_chips.py
test_server_errors.py test_web_columns.pyThe bug likely involves a table view misalignment issue—I'll check the repository structure.
root@3f0164aeb2f9:/workspace/scubaduck# ls scubaduck
__init__.py sample.csv server.py staticI'll check out server.py to locate the function for running queries.
root@3f0164aeb2f9:/workspace/scubaduck# sed -n '1,160p' scubaduck/server.py
from __future__ import annotations
from dataclasses import dataclass, field, replace
from typing import Any, Dict, List, Tuple, cast
import re
from datetime import datetime, timedelta, timezone
import time
from pathlib import Path
import os
import traceback
import math
import duckdb
from dateutil import parser as dtparser
from dateutil.relativedelta import relativedelta
from flask import Flask, jsonify, request, send_from_directory
def _quote(ident: str) -> str:
"""Return identifier quoted for SQL."""
return f'"{ident.replace('"', '""')}"'
@dataclass
class Filter:
column: str
op: str
value: str | int | float | list[str] | None
@dataclass
class QueryParams:
start: str | None = None
end: str | None = None
order_by: str | None = None
order_dir: str = "ASC"
limit: int | None = None
columns: list[str] = field(default_factory=lambda: [])
filters: list[Filter] = field(default_factory=lambda: [])
derived_columns: dict[str, str] = field(default_factory=lambda: {})
graph_type: str = "samples"
group_by: list[str] = field(default_factory=lambda: [])
aggregate: str | None = None
show_hits: bool = False
x_axis: str | None = None
granularity: str = "Auto"
fill: str = "0"
table: str = "events"
time_column: str | None = "timestamp"
time_unit: str = "s"
def _load_database(path: Path) -> duckdb.DuckDBPyConnection:
if not path.exists():
raise FileNotFoundError(path)
ext = path.suffix.lower()
if ext == ".csv":
con = duckdb.connect()
con.execute(
f"CREATE TABLE events AS SELECT * FROM read_csv_auto('{path.as_posix
()}')"
)
elif ext in {".db", ".sqlite"}:
con = duckdb.connect()
con.execute("LOAD sqlite")
con.execute(f"ATTACH '{path.as_posix()}' AS db (TYPE SQLITE)")
tables = [
r[0]
for r in con.execute(
"SELECT name FROM sqlite_master WHERE type='table'"
).fetchall()
]
for t in tables:
con.execute(f'CREATE VIEW "{t}" AS SELECT * FROM db."{t}"')
else:
con = duckdb.connect(path)
return con
def _create_test_database() -> duckdb.DuckDBPyConnection:
"""Return a DuckDB connection with a small multi-table dataset."""
con = duckdb.connect()
con.execute(
"CREATE TABLE events (id INTEGER PRIMARY KEY, ts INTEGER, val REAL, name
TEXT, flag BOOLEAN)"
)
con.execute("INSERT INTO events VALUES (1, 1704067200, 1.5, 'alice', 1)")
con.execute("INSERT INTO events VALUES (2, 1704070800, 2.0, 'bob', 0)")
con.execute('CREATE TABLE extra (ts INTEGER, "desc" TEXT, num INTEGER)')
con.execute("INSERT INTO extra VALUES (1704067200, 'x', 1)")
con.execute("INSERT INTO extra VALUES (1704070800, 'y', 2)")
return con
_REL_RE = re.compile(
r"([+-]?\d+(?:\.\d*)?)\s*(hour|hours|day|days|week|weeks|fortnight|fortnight
s|month|months|year|years)",
re.IGNORECASE,
)
def parse_time(val: str | None) -> str | None:
"""Parse an absolute or relative time string into ``YYYY-MM-DD HH:MM:SS``.""
"
if val is None or val == "":
return None
s = val.strip()
if s.lower() == "now":
dt = datetime.now(timezone.utc)
return dt.replace(microsecond=0).strftime("%Y-%m-%d %H:%M:%S")
m = _REL_RE.fullmatch(s)
if m:
qty = float(m.group(1))
unit = m.group(2).lower()
now = datetime.now(timezone.utc)
dt: datetime
if unit.startswith("hour"):
dt = now + timedelta(hours=qty)
elif unit.startswith("day"):
dt = now + timedelta(days=qty)
elif unit.startswith("week"):
dt = now + timedelta(weeks=qty)
elif unit.startswith("fortnight"):
dt = now + timedelta(weeks=2 * qty)
elif unit.startswith("month"):
if qty.is_integer():
dt = now + relativedelta(months=int(qty))
else:
dt = now + timedelta(days=30 * qty)
elif unit.startswith("year"):
if qty.is_integer():
dt = now + relativedelta(years=int(qty))
else:
dt = now + timedelta(days=365 * qty)
else: # pragma: no cover - defensive
raise ValueError(f"Unsupported unit: {unit}")
return dt.replace(microsecond=0).strftime("%Y-%m-%d %H:%M:%S")
dt = dtparser.parse(s)
return dt.replace(microsecond=0, tzinfo=None).strftime("%Y-%m-%d %H:%M:%S")
def _numeric_to_datetime(value: int | float, unit: str) -> datetime:
"""Convert a numeric timestamp ``value`` with unit ``unit`` to ``datetime``.
Heuristically fall back to seconds when the converted value is before 1990
but the seconds interpretation is in a reasonable range. This handles
integer columns stored in seconds even when ``unit`` is mistakenly set to a
finer granularity.
"""
divisor = {
"s": 1,
"ms": 1000,
"us": 1_000_000,
"ns": 1_000_000_000,
}.get(unit, 1)
dt = datetime.fromtimestamp(int(value) / divisor, tz=timezone.utc)
if unit != "s" and dt.year < 1990:Since server.py is long, I'll look for lines involving "group by" or "count".
root@3f0164aeb2f9:/workspace/scubaduck# grep -n "group" -n scubaduck/server.py |
head -n 20
44: group_by: list[str] = field(default_factory=lambda: [])
113: qty = float(m.group(1))
114: unit = m.group(2).lower()
250: group_cols = params.group_by[:]
269: group_cols = ["bucket"] + group_cols
271: has_agg = bool(group_cols) or params.aggregate is not None
274: group_cols[1:] if params.graph_type == "timeseries" else group_c
ols
278: selected_for_order.update(group_cols)
303: if col in group_cols:
307: select_parts.insert(len(group_cols), "count(*) AS Hits")
388: if group_cols:
389: lines.append("GROUP BY " + ", ".join(_quote(c) for c in group_cols))
508: group_by=payload.get("group_by", []),
535: params.group_by or params.aggregate or params.show_hits
540: "error": "group_by, aggregate and show_hits are only
valid for table or timeseries view"
573: for col in params.group_by:
579: if params.group_by or params.graph_type == "timeseries":
592: if c in params.group_by or c == params.x_axis:I’ll search for mentions of 'aggregate count' to gather relevant information.
root@3f0164aeb2f9:/workspace/scubaduck# grep -n "aggregate" -n scubaduck/server.
py | head -n 20
45: aggregate: str | None = None
271: has_agg = bool(group_cols) or params.aggregate is not None
277: agg = (params.aggregate or "count").lower()
509: aggregate=payload.get("aggregate"),
535: params.group_by or params.aggregate or params.show_hits
540: "error": "group_by, aggregate and show_hits are only
valid for table or timeseries view"
580: agg = (params.aggregate or "count").lower()I'm scrolling through the lines to check the necessary content.
root@3f0164aeb2f9:/workspace/scubaduck# sed -n '248,316p' scubaduck/server.py
def build_query(params: QueryParams, column_types: Dict[str, str] | None = None)
-> str:
select_parts: list[str] = []
group_cols = params.group_by[:]
selected_for_order = set(params.columns) | set(params.derived_columns.keys()
)
if params.graph_type == "timeseries":
sec = _granularity_seconds(params.granularity, params.start, params.end)
x_axis = params.x_axis or params.time_column
if x_axis is None:
raise ValueError("x_axis required for timeseries")
xexpr = _time_expr(x_axis, column_types, params.time_unit)
if params.start:
bucket_expr = (
f"TIMESTAMP '{params.start}' + INTERVAL '{sec} second' * "
f"CAST(floor((epoch({xexpr}) - epoch(TIMESTAMP '{params.start}')
)/{sec}) AS BIGINT)"
)
else:
bucket_expr = (
f"TIMESTAMP 'epoch' + INTERVAL '{sec} second' * "
f"CAST(floor(epoch({xexpr})/{sec}) AS BIGINT)"
)
select_parts.append(f"{bucket_expr} AS bucket")
group_cols = ["bucket"] + group_cols
selected_for_order.add("bucket")
has_agg = bool(group_cols) or params.aggregate is not None
if has_agg:
select_cols = (
group_cols[1:] if params.graph_type == "timeseries" else group_cols
)
select_parts.extend(_quote(c) for c in select_cols)
agg = (params.aggregate or "count").lower()
selected_for_order.update(group_cols)
def agg_expr(col: str) -> str:
expr = _quote(col)
ctype = column_types.get(col, "").upper() if column_types else ""
if "BOOL" in ctype:
expr = f"CAST({_quote(col)} AS BIGINT)"
if agg.startswith("p"):
quant = float(agg[1:]) / 100
return f"quantile({expr}, {quant})"
if agg == "count distinct":
return f"count(DISTINCT {expr})"
if agg == "avg" and column_types is not None:
if "TIMESTAMP" in ctype or "DATE" in ctype or "TIME" in ctype:
return (
"TIMESTAMP 'epoch' + INTERVAL '1 second' * "
f"CAST(avg(epoch({_quote(col)})) AS BIGINT)"
)
return f"{agg}({expr})"
if agg == "count":
select_parts.append("count(*) AS Count")
selected_for_order.add("Count")
else:
for col in params.columns:
if col in group_cols:
continue
select_parts.append(f"{agg_expr(col)} AS {_quote(col)}")
selected_for_order.add(col)
select_parts.insert(len(group_cols), "count(*) AS Hits")
selected_for_order.add("Hits")
else:
select_parts.extend(_quote(c) for c in params.columns)
selected_for_order.update(params.columns)
order_by = params.order_by
if order_by == "Samples":
order_by = "Hits"
order_by = order_by if order_by in selected_for_order else NoneI'll keep going until I reach the section around line 316.
root@3f0164aeb2f9:/workspace/scubaduck# sed -n '316,408p' scubaduck/server.py
order_by = order_by if order_by in selected_for_order else None
if has_agg and params.derived_columns:
inner_params = replace(
params,
derived_columns={},
order_by=None,
limit=None,
)
inner_sql = build_query(inner_params, column_types)
outer_select = ["t.*"] + [
f"{expr} AS {name}" for name, expr in params.derived_columns.items()
]
indented_inner = "\n".join(" " + line for line in inner_sql.splitline
s())
lines = [
f"SELECT {', '.join(outer_select)}",
"FROM (",
indented_inner,
") t",
]
if order_by:
lines.append(f"ORDER BY {_quote(order_by)} {params.order_dir}")
elif params.graph_type == "timeseries":
lines.append("ORDER BY bucket")
if params.limit is not None:
lines.append(f"LIMIT {params.limit}")
return "\n".join(lines)
for name, expr in params.derived_columns.items():
select_parts.append(f"{expr} AS {name}")
selected_for_order.add(name)
select_clause = ", ".join(select_parts) if select_parts else "*"
lines = [f"SELECT {select_clause}", f'FROM "{params.table}"']
where_parts: list[str] = []
if params.time_column:
time_expr = _time_expr(params.time_column, column_types, params.time_uni
t)
else:
time_expr = None
if time_expr and params.start:
where_parts.append(f"{time_expr} >= '{params.start}'")
if time_expr and params.end:
where_parts.append(f"{time_expr} <= '{params.end}'")
for f in params.filters:
op = f.op
if op in {"empty", "!empty"}:
val = "''"
else:
if f.value is None:
continue
if isinstance(f.value, list):
if not f.value:
continue
if op == "=":
qcol = _quote(f.column)
vals = " OR ".join(f"{qcol} = '{v}'" for v in f.value)
where_parts.append(f"({vals})")
continue
val = f"'{f.value}'" if isinstance(f.value, str) else str(f.value)
qcol = _quote(f.column)
if op == "contains":
where_parts.append(f"{qcol} ILIKE '%' || {val} || '%'")
elif op == "!contains":
where_parts.append(f"{qcol} NOT ILIKE '%' || {val} || '%'")
elif op == "empty":
where_parts.append(f"{qcol} = {val}")
elif op == "!empty":
where_parts.append(f"{qcol} != {val}")
else:
where_parts.append(f"{qcol} {op} {val}")
if where_parts:
lines.append("WHERE " + " AND ".join(where_parts))
if group_cols:
lines.append("GROUP BY " + ", ".join(_quote(c) for c in group_cols))
if order_by:
lines.append(f"ORDER BY {_quote(order_by)} {params.order_dir}")
elif params.graph_type == "timeseries":
lines.append("ORDER BY bucket")
if params.limit is not None:
lines.append(f"LIMIT {params.limit}")
return "\n".join(lines)
def create_app(db_file: str | Path | None = None) -> Flask:
app = Flask(__name__, static_folder="static")
if db_file is None:
env_db = os.environ.get("SCUBADUCK_DB")
if env_db:
db_file = env_db
if isinstance(db_file, str) and db_file.upper() == "TEST":
con = _create_test_database()
else:
db_path = Path(db_file or Path(__file__).with_name("sample.csv")).resolv
e()I'll keep scrolling until I reach section 408 to gather the necessary details.
root@3f0164aeb2f9:/workspace/scubaduck# sed -n '408,612p' scubaduck/server.py
db_path = Path(db_file or Path(__file__).with_name("sample.csv")).resolv
e()
con = _load_database(db_path)
tables = [r[0] for r in con.execute("SHOW TABLES").fetchall()]
if not tables:
raise ValueError("No tables found in database")
default_table = tables[0]
columns_cache: Dict[str, Dict[str, str]] = {}
def get_columns(table: str) -> Dict[str, str]:
if table not in columns_cache:
rows = con.execute(f'PRAGMA table_info("{table}")').fetchall()
if not rows:
raise ValueError(f"Unknown table: {table}")
columns_cache[table] = {r[1]: r[2] for r in rows}
return columns_cache[table]
sample_cache: Dict[Tuple[str, str, str], Tuple[List[str], float]] = {}
CACHE_TTL = 60.0
CACHE_LIMIT = 200
@app.route("/")
def index() -> Any: # pyright: ignore[reportUnusedFunction]
assert app.static_folder is not None
return send_from_directory(app.static_folder, "index.html")
@app.route("/js/<path:filename>")
def js(filename: str) -> Any: # pyright: ignore[reportUnusedFunction]
assert app.static_folder is not None
folder = Path(app.static_folder) / "js"
return send_from_directory(folder, filename)
@app.route("/api/tables")
def tables_endpoint() -> Any: # pyright: ignore[reportUnusedFunction]
return jsonify(tables)
@app.route("/api/columns")
def columns() -> Any: # pyright: ignore[reportUnusedFunction]
table = request.args.get("table", default_table)
rows = con.execute(f'PRAGMA table_info("{table}")').fetchall()
return jsonify([{"name": r[1], "type": r[2]} for r in rows])
def _cache_get(key: Tuple[str, str, str]) -> List[str] | None:
item = sample_cache.get(key)
if item is None:
return None
vals, ts = item
if time.time() - ts > CACHE_TTL:
del sample_cache[key]
return None
sample_cache[key] = (vals, time.time())
return vals
def _cache_set(key: Tuple[str, str, str], vals: List[str]) -> None:
sample_cache[key] = (vals, time.time())
if len(sample_cache) > CACHE_LIMIT:
oldest = min(sample_cache.items(), key=lambda kv: kv[1][1])[0]
del sample_cache[oldest]
@app.route("/api/samples")
def sample_values() -> Any: # pyright: ignore[reportUnusedFunction]
table = request.args.get("table", default_table)
column = request.args.get("column")
substr = request.args.get("q", "")
column_types = get_columns(table)
if not column or column not in column_types:
return jsonify([])
ctype = column_types[column].upper()
if "CHAR" not in ctype and "STRING" not in ctype and "VARCHAR" not in ct
ype:
return jsonify([])
key = (table, column, substr)
cached = _cache_get(key)
if cached is not None:
return jsonify(cached)
qcol = _quote(column)
rows = con.execute(
f"SELECT DISTINCT {qcol} FROM \"{table}\" WHERE CAST({qcol} AS VARCH
AR) ILIKE '%' || ? || '%' LIMIT 20",
[substr],
).fetchall()
values = [r[0] for r in rows]
_cache_set(key, values)
return jsonify(values)
@app.route("/api/query", methods=["POST"])
def query() -> Any: # pyright: ignore[reportUnusedFunction]
payload = request.get_json(force=True)
try:
start = parse_time(payload.get("start"))
end = parse_time(payload.get("end"))
except Exception as exc:
return jsonify({"error": str(exc)}), 400
params = QueryParams(
start=start,
end=end,
order_by=payload.get("order_by"),
order_dir=payload.get("order_dir", "ASC"),
limit=payload.get("limit"),
columns=payload.get("columns", []),
derived_columns=payload.get("derived_columns", {}),
graph_type=payload.get("graph_type", "samples"),
group_by=payload.get("group_by", []),
aggregate=payload.get("aggregate"),
show_hits=payload.get("show_hits", False),
x_axis=payload.get("x_axis"),
granularity=payload.get("granularity", "Auto"),
fill=payload.get("fill", "0"),
table=payload.get("table", default_table),
time_column=payload.get("time_column", "timestamp"),
time_unit=payload.get("time_unit", "s"),
)
if params.order_by == "Samples":
params.order_by = "Hits"
for f in payload.get("filters", []):
params.filters.append(Filter(f["column"], f["op"], f.get("value")))
if params.table not in tables:
return jsonify({"error": "Invalid table"}), 400
column_types = get_columns(params.table)
if params.time_column and params.time_column not in column_types:
return jsonify({"error": "Invalid time_column"}), 400
if params.time_unit not in {"s", "ms", "us", "ns"}:
return jsonify({"error": "Invalid time_unit"}), 400
if params.graph_type not in {"table", "timeseries"} and (
params.group_by or params.aggregate or params.show_hits
):
return (
jsonify(
{
"error": "group_by, aggregate and show_hits are only val
id for table or timeseries view"
}
),
400,
)
valid_cols = set(column_types.keys())
valid_cols.update(params.derived_columns.keys())
valid_cols.add("Hits")
if params.graph_type == "timeseries":
if params.x_axis is None:
params.x_axis = params.time_column
if params.x_axis is None or params.x_axis not in valid_cols:
return jsonify({"error": "Invalid x_axis"}), 400
ctype = column_types.get(params.x_axis, "").upper()
is_time = any(t in ctype for t in ["TIMESTAMP", "DATE", "TIME"])
is_numeric = any(
t in ctype
for t in [
"INT",
"DECIMAL",
"REAL",
"DOUBLE",
"FLOAT",
"NUMERIC",
"HUGEINT",
]
)
if not (is_time or is_numeric):
return jsonify({"error": "x_axis must be a time column"}), 400
for col in params.columns:
if col not in valid_cols:
return jsonify({"error": f"Unknown column: {col}"}), 400
for col in params.group_by:
if col not in valid_cols:
return jsonify({"error": f"Unknown column: {col}"}), 400
if params.order_by and params.order_by not in valid_cols:
return jsonify({"error": f"Unknown column: {params.order_by}"}), 400
if params.group_by or params.graph_type == "timeseries":
agg = (params.aggregate or "count").lower()
if agg.startswith("p") or agg == "sum":
need_numeric = True
allow_time = False
elif agg == "avg" or agg in {"min", "max"}:
need_numeric = False
allow_time = True
else:
need_numeric = False
allow_time = False
if need_numeric or allow_time:
for c in params.columns:
if c in params.group_by or c == params.x_axis:
continue
if c not in column_types:
continue
ctype = column_types.get(c, "").upper()
is_numeric = "BOOL" in ctype or any(
t in ctype
for t in [
"INT",
"DECIMAL",
"REAL",
"DOUBLE",
"FLOAT",
"NUMERIC",
"HUGEINT",
]
)
is_time = "TIMESTAMP" in ctype or "DATE" in ctype or "TIME"
in ctype
if need_numeric and not is_numeric:
return (
jsonify(I need to continue scrolling, aiming for around line 612.
root@3f0164aeb2f9:/workspace/scubaduck# sed -n '612,872p' scubaduck/server.py
jsonify(
{
"error": f"Aggregate {agg} cannot be applied
to column {c}",
}
),
400,
)
if allow_time and not (is_numeric or is_time):
return (
jsonify(
{
"error": f"Aggregate {agg} cannot be applied
to column {c}",
}
),
400,
)
if (params.start is None or params.end is None) and (
params.x_axis or params.time_column
):
axis = params.x_axis or params.time_column
assert axis is not None
row = cast(
tuple[datetime | None, datetime | None],
con.execute(
f'SELECT min({_quote(axis)}), max({_quote(axis)}) FROM "{par
ams.table}"'
).fetchall()[0],
)
mn, mx = row
if isinstance(mn, (int, float)):
try:
mn = _numeric_to_datetime(mn, params.time_unit)
except Exception:
return (
jsonify(
{
"error": (
f"Invalid time value {mn} for column {axis}"
f" with time_unit {params.time_unit}"
)
}
),
400,
)
if isinstance(mx, (int, float)):
try:
mx = _numeric_to_datetime(mx, params.time_unit)
except Exception:
return (
jsonify(
{
"error": (
f"Invalid time value {mx} for column {axis}"
f" with time_unit {params.time_unit}"
)
}
),
400,
)
if params.start is None and mn is not None:
params.start = (
mn.strftime("%Y-%m-%d %H:%M:%S") if not isinstance(mn, str)
else mn
)
if params.end is None and mx is not None:
params.end = (
mx.strftime("%Y-%m-%d %H:%M:%S") if not isinstance(mx, str)
else mx
)
bucket_size: int | None = None
if params.graph_type == "timeseries":
bucket_size = _granularity_seconds(
params.granularity,
params.start if isinstance(params.start, str) else None,
params.end if isinstance(params.end, str) else None,
)
if (
params.limit is not None
and params.start is not None
and params.end is not None
):
try:
start_dt = dtparser.parse(params.start)
end_dt = dtparser.parse(params.end)
buckets = math.ceil(
(end_dt - start_dt).total_seconds() / bucket_size
)
if buckets > 1:
params.limit *= buckets
except Exception:
pass
sql = build_query(params, column_types)
try:
rows = con.execute(sql).fetchall()
except Exception as exc:
tb = traceback.format_exc()
print(f"Query failed:\n{sql}\n{tb}")
return (
jsonify({"sql": sql, "error": str(exc), "traceback": tb}),
400,
)
result: Dict[str, Any] = {"sql": sql, "rows": rows}
if params.start is not None:
result["start"] = str(params.start)
if params.end is not None:
result["end"] = str(params.end)
if bucket_size is not None:
result["bucket_size"] = bucket_size
return jsonify(result)
return app
app = create_app()
if __name__ == "__main__":
app.run(debug=True)I’m going to check out the tests in test_server.py to confirm the expected behavior.
root@3f0164aeb2f9:/workspace/scubaduck# ls tests | grep server
test_server.py
test_server_basic.py
test_server_db_types.py
test_server_errors.py
test_server_time.py
test_server_timeseries.pyI’ll review the tests to get a better understanding of the features.
root@3f0164aeb2f9:/workspace/scubaduck# sed -n '1,160p' tests/test_server.py
"""Legacy test module.
Tests have been split into multiple files:
- test_server_basic.py
- test_server_db_types.py
- test_server_time.py
- test_server_errors.py
- test_server_timeseries.py
"""root@3f0164aeb2f9:/workspace/scubaduck# sed -n '1,200p' tests/test_server_basic.
py
from __future__ import annotations
import json
from scubaduck import server
def test_basic_query() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-02 00:00:00",
"order_by": "timestamp",
"order_dir": "ASC",
"limit": 10,
"columns": ["timestamp", "event", "value", "user"],
"filters": [],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert data
rows = data["rows"]
# We expect first three rows (until 2024-01-02 00:00:00)
assert len(rows) == 3
assert rows[0][1] == "login"
assert rows[1][1] == "logout"
def test_js_served() -> None:
app = server.app
client = app.test_client()
rv = client.get("/js/chip_input.js")
assert rv.status_code == 200
assert b"initChipInput" in rv.data
def test_filter_multi_token() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-02 03:00:00",
"order_by": "timestamp",
"limit": 10,
"columns": ["timestamp", "event", "value", "user"],
"filters": [{"column": "user", "op": "=", "value": ["alice", "charlie"]}
],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert data
rows = data["rows"]
# Should only return rows for alice and charlie
assert len(rows) == 3
assert rows[0][3] == "alice"
assert rows[-1][3] == "charlie"
def test_empty_filter_is_noop() -> None:
app = server.app
client = app.test_client()
base_payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"limit": 100,
"columns": ["timestamp", "event", "value", "user"],
}
no_filter = {**base_payload, "filters": []}
empty_filter = {
**base_payload,
"filters": [{"column": "user", "op": "=", "value": None}],
}
rv1 = client.post(
"/api/query", data=json.dumps(no_filter), content_type="application/json
"
)
rv2 = client.post(
"/api/query", data=json.dumps(empty_filter), content_type="application/j
son"
)
rows1 = rv1.get_json()["rows"]
rows2 = rv2.get_json()["rows"]
assert rows1 == rows2
def test_select_columns() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"order_by": "timestamp",
"limit": 10,
"columns": ["timestamp", "user"],
"filters": [],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert data
rows = data["rows"]
assert len(rows[0]) == 2
assert rows[0][1] == "alice"
def test_string_filter_ops() -> None:
app = server.app
client = app.test_client()
base = {
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"order_by": "timestamp",
"limit": 100,
"columns": ["timestamp", "event", "value", "user"],
}
contains = {
**base,
"filters": [{"column": "user", "op": "contains", "value": "ali"}],
}
rv = client.post(
"/api/query", data=json.dumps(contains), content_type="application/json"
)
rows = rv.get_json()["rows"]
assert all("ali" in r[3] for r in rows)
regex = {
**base,
"filters": [{"column": "user", "op": "~", "value": "^a.*"}],
}
rv = client.post(
"/api/query", data=json.dumps(regex), content_type="application/json"
)
rows = rv.get_json()["rows"]
assert all(r[3].startswith("a") for r in rows)
assert len(rows) == 2
not_empty = {**base, "filters": [{"column": "user", "op": "!empty"}]}
rv = client.post(
"/api/query", data=json.dumps(not_empty), content_type="application/json
"
)
assert len(rv.get_json()["rows"]) == 4
def test_order_by_ignored_when_not_selected() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"order_by": "value",
"columns": ["timestamp"],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert "ORDER BY" not in data["sql"]I'll explore the test_server_basic further and check around the group-by logic.
root@3f0164aeb2f9:/workspace/scubaduck# grep -n "group_by" -n tests/test_server_
basic.pyI’ll search all tests for the group_by to check if it's used anywhere else.
tests/test_server_errors.py:36: "group_by": ["user"],
tests/test_server_errors.py:58: "group_by": ["user"],
tests/test_server_errors.py:69:def test_samples_view_rejects_group_by() -> None:
tests/test_server_errors.py:75: "group_by": ["user"],
tests/test_web_table.py:77:def test_table_avg_group_by(page: Any, server_url: st
r) -> None:
tests/test_web_table.py:84: group_by=["user"],
tests/test_web_table.py:179:def test_group_by_chip_from_url(page: Any, server_ur
l: str) -> None:
tests/test_web_table.py:180: url = f"{server_url}?graph_type=table&group_by=u
ser&order_by=user&limit=10"
tests/test_web_table.py:182: page.wait_for_selector("#group_by_field .chip",
state="attached")
tests/test_web_table.py:184: "Array.from(document.querySelectorAll('#grou
p_by_field .chip')).map(c => c.firstChild.textContent)"
tests/test_web_table.py:189:def test_group_by_autocomplete(page: Any, server_url
: str) -> None:
tests/test_web_table.py:191: page.wait_for_selector("#group_by_field", state=
"visible")
tests/test_web_table.py:192: inp = page.query_selector("#group_by_field .f-va
l")
tests/test_web_table.py:196: page.wait_for_selector("#group_by_field .chip-dr
opdown div")
tests/test_web_table.py:197: options = page.locator("#group_by_field .chip-dr
opdown div").all_inner_texts()
tests/test_web_table.py:201:def test_group_by_copy_icon(page: Any, server_url: s
tr) -> None:
tests/test_web_table.py:203: page.wait_for_selector("#group_by_field", state=
"visible")
tests/test_web_table.py:204: icon = page.text_content("#group_by_field .chip-
copy")
tests/test_web_table.py:208:def test_group_by_input_no_border(page: Any, server_
url: str) -> None:
tests/test_web_table.py:210: page.wait_for_selector("#group_by_field", state=
"visible")I'm continuing to scroll through the test cases to check for group_by testing.
import json
from scubaduck import server
def test_invalid_time_error() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "nonsense",
"end": "now",
"order_by": "timestamp",
"limit": 10,
"columns": ["timestamp"],
"filters": [],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 400
assert "error" in data
def test_query_error_returns_sql_and_traceback() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"columns": ["event"],
"group_by": ["user"],
"aggregate": "avg",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 400
assert "error" in data
def test_table_unknown_column_error() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "table",
"order_by": "timestamp",
"limit": 100,
"columns": ["user", "Hits", "value"],
"group_by": ["user"],
"aggregate": "Count",
"show_hits": True,
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
def test_samples_view_rejects_group_by() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"graph_type": "samples",
"group_by": ["user"],
"columns": ["timestamp"],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 400
assert "only valid" in data["error"]I'm continuing to scroll through the rest of the file.
root@3f0164aeb2f9:/workspace/scubaduck# sed -n '160,320p' tests/test_server_erro
rs.pyI’ll open tests/test_server_db_types to check out the rest.
import duckdb
import pytest
from scubaduck import server
def _make_payload() -> dict[str, object]:
return {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-02 00:00:00",
"order_by": "timestamp",
"order_dir": "ASC",
"limit": 10,
"columns": ["timestamp", "event", "value", "user"],
"filters": [],
}
def test_database_types(tmp_path: Path) -> None:
csv_file = tmp_path / "events.csv"
csv_file.write_text(Path("scubaduck/sample.csv").read_text())
sqlite_file = tmp_path / "events.sqlite"
import sqlite3
conn = sqlite3.connect(sqlite_file)
conn.execute(
"CREATE TABLE events (timestamp TEXT, event TEXT, value INTEGER, user TE
XT)"
)
with open(csv_file) as f:
next(f)
for line in f:
ts, ev, val, user = line.strip().split(",")
conn.execute(
"INSERT INTO events VALUES (?, ?, ?, ?)", (ts, ev, int(val), use
r)
)
conn.commit()
conn.close() # pyright: ignore[reportUnknownMemberType, reportAttributeAcce
ssIssue]
duckdb_file = tmp_path / "events.duckdb"
con = duckdb.connect(duckdb_file)
con.execute(
f"CREATE TABLE events AS SELECT * FROM read_csv_auto('{csv_file.as_posix
()}')"
)
con.close() # pyright: ignore[reportUnknownMemberType, reportAttributeAcces
sIssue]
for db in (csv_file, sqlite_file, duckdb_file):
app = server.create_app(db)
client = app.test_client()
payload = _make_payload()
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/js
on"
)
rows = rv.get_json()["rows"]
assert len(rows) == 3
def test_sqlite_longvarchar(tmp_path: Path) -> None:
sqlite_file = tmp_path / "events.sqlite"
import sqlite3
conn = sqlite3.connect(sqlite_file)
conn.execute(
"CREATE TABLE events (timestamp TEXT, url LONGVARCHAR, title VARCHAR(10)
)"
)
conn.execute(
"INSERT INTO events VALUES ('2024-01-01 00:00:00', 'https://a.com', 'Hom
e')"
)
conn.commit()
conn.close() # pyright: ignore[reportUnknownMemberType, reportAttributeAcce
ssIssue]
app = server.create_app(sqlite_file)
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-01 01:00:00",
"order_by": "timestamp",
"columns": ["timestamp", "url", "title"],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert data["rows"][0][1] == "https://a.com"
def test_sqlite_bigint(tmp_path: Path) -> None:
sqlite_file = tmp_path / "big.sqlite"
import sqlite3
conn = sqlite3.connect(sqlite_file)
conn.execute("CREATE TABLE events (timestamp TEXT, value INTEGER)")
big_value = 13385262862605259
conn.execute(
"INSERT INTO events VALUES ('2024-01-01 00:00:00', ?)",
(big_value,),
)
conn.commit()
conn.close() # pyright: ignore[reportUnknownMemberType, reportAttributeAcce
ssIssue]
app = server.create_app(sqlite_file)
client = app.test_client()
payload = {
"table": "events",
"order_by": "timestamp",
"columns": ["timestamp", "value"],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert data["rows"][0][1] == big_value
def test_sqlite_boolean_aggregation(tmp_path: Path) -> None:
sqlite_file = tmp_path / "bool.sqlite"
import sqlite3
conn = sqlite3.connect(sqlite_file)
conn.execute("CREATE TABLE events (timestamp TEXT, flag BOOLEAN)")
conn.execute("INSERT INTO events VALUES ('2024-01-01 00:00:00', 1)")
conn.execute("INSERT INTO events VALUES ('2024-01-01 00:30:00', 0)")
conn.commit()
conn.close() # pyright: ignore[reportUnknownMemberType, reportAttributeAcce
ssIssue]
app = server.create_app(sqlite_file)
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-02 00:00:00",
"graph_type": "table",
"aggregate": "Avg",
"columns": ["flag"],
"show_hits": True,
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert data["rows"][0][0] == 2
assert data["rows"][0][1] == 0.5
def test_sqlite_boolean_group_by(tmp_path: Path) -> None:
sqlite_file = tmp_path / "bool.sqlite"
import sqlite3
conn = sqlite3.connect(sqlite_file)
conn.execute("CREATE TABLE events (id INTEGER, ts TEXT, flag BOOLEAN)")
conn.execute("INSERT INTO events VALUES (1, '2024-01-01 00:00:00', 1)")
conn.execute("INSERT INTO events VALUES (1, '2024-01-01 00:30:00', 0)")
conn.execute("INSERT INTO events VALUES (2, '2024-01-01 01:00:00', 1)")
conn.commit()
conn.close() # pyright: ignore[reportUnknownMemberType, reportAttributeAcce
ssIssue]
app = server.create_app(sqlite_file)
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-02 00:00:00",
"graph_type": "table",
"time_column": "ts",
"aggregate": "Avg",
"group_by": ["id"],
"columns": ["flag"],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
rows = sorted(data["rows"]) # order can vary
assert rows == [[1, 2, 0.5], [2, 1, 1.0]]
def test_envvar_db(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
csv_file = tmp_path / "custom.csv"
csv_file.write_text("timestamp,event,value,user\n2024-01-01 00:00:00,login,5
,bob\n")
monkeypatch.setenv("SCUBADUCK_DB", str(csv_file))
app = server.create_app()
client = app.test_client()
payload = _make_payload()
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
rows = rv.get_json()["rows"]
assert len(rows) == 1I'm continuing to scroll down until I reach the necessary section at the bottom.
ypes.py
def test_envvar_db_missing(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> N
one:
missing = tmp_path / "missing.sqlite"
monkeypatch.setenv("SCUBADUCK_DB", str(missing))
with pytest.raises(FileNotFoundError):
server.create_app()from scubaduck import server
def test_integer_time_column(tmp_path: Path) -> None:
csv_file = tmp_path / "events.csv"
csv_file.write_text("created,event\n1704067200,login\n1704070800,logout\n")
app = server.create_app(csv_file)
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-01 01:00:00",
"order_by": "created",
"columns": ["created", "event"],
"time_column": "created",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert len(data["rows"]) == 2
def test_integer_time_unit_ms(tmp_path: Path) -> None:
csv_file = tmp_path / "events.csv"
csv_file.write_text("created,event\n1704067200000,login\n1704070800000,logou
t\n")
app = server.create_app(csv_file)
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-01 01:00:00",
"order_by": "created",
"columns": ["created", "event"],
"time_column": "created",
"time_unit": "ms",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert len(data["rows"]) == 2
def test_timeseries_default_xaxis_uses_time_column(tmp_path: Path) -> None:
csv_file = tmp_path / "events.csv"
csv_file.write_text("created,event\n1704067200000,login\n1704070800000,logou
t\n")
app = server.create_app(csv_file)
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-01 01:00:00",
"graph_type": "timeseries",
"granularity": "1 hour",
"columns": ["event"],
"aggregate": "Count",
"time_column": "created",
"time_unit": "ms",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert len(data["rows"]) == 2
def test_integer_time_unit_us_default_start_end(tmp_path: Path) -> None:
csv_file = tmp_path / "events.csv"
csv_file.write_text(
"created,event\n1704067200000000,login\n1704070800000000,logout\n"
)
app = server.create_app(csv_file)
client = app.test_client()
payload = {
"table": "events",
"order_by": "created",
"columns": ["created", "event"],
"time_column": "created",
"time_unit": "us",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert data["start"] == "2024-01-01 00:00:00"
assert data["end"] == "2024-01-01 01:00:00"
assert len(data["rows"]) == 2
def test_sqlite_integer_time_unit_us(tmp_path: Path) -> None:
sqlite_file = tmp_path / "events.sqlite"
import sqlite3
conn = sqlite3.connect(sqlite_file)
conn.execute("CREATE TABLE visits (visit_time INTEGER, event TEXT)")
big_ts = 13384551652000000
conn.execute("INSERT INTO visits VALUES (?, ?)", (big_ts, "foo"))
conn.commit()
conn.close() # pyright: ignore[reportUnknownMemberType, reportAttributeAcce
ssIssue]
app = server.create_app(sqlite_file)
client = app.test_client()
payload = {
"table": "visits",
"start": "2394-02-20 00:00:00",
"end": "2394-02-21 00:00:00",
"order_by": "visit_time",
"columns": ["visit_time", "event"],
"time_column": "visit_time",
"time_unit": "us",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert len(data["rows"]) == 1
def test_relative_time_query(monkeypatch: pytest.MonkeyPatch) -> None:
app = server.app
client = app.test_client()
from datetime import datetime
fixed_now = datetime(2024, 1, 2, 4, 0, 0)
class FixedDateTime(datetime):
@classmethod
def now(cls, tz=None): # type: ignore[override]
return fixed_now if tz is None else fixed_now.astimezone(tz)
monkeypatch.setattr(server, "datetime", FixedDateTime)
payload = {
"table": "events",
"start": "-1 hour",
"end": "now",
"order_by": "timestamp",
"limit": 100,
"columns": ["timestamp", "event", "value", "user"],
"filters": [],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert len(data["rows"]) == 1
assert data["rows"][0][3] == "charlie"
def test_relative_month_year(monkeypatch: pytest.MonkeyPatch) -> None:
app = server.app
client = app.test_client()
from datetime import datetime
fixed_now = datetime(2024, 1, 2, 0, 0, 0)
class FixedDateTime(datetime):
@classmethod
def now(cls, tz=None): # type: ignore[override]
return fixed_now if tz is None else fixed_now.astimezone(tz)
monkeypatch.setattr(server, "datetime", FixedDateTime)
payload = {
"table": "events",
"start": "-1 year",
"end": "-1 month",
"order_by": "timestamp",
"limit": 10,
"columns": ["timestamp"],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert data["start"] == "2023-01-02 00:00:00"
assert data["end"] == "2023-12-02 00:00:00"
def test_default_start_end_returned() -> None:
app = server.app
client = app.test_client()Let's continue scrolling to the file's bottom to check for groupby usage.
payload = {
"table": "events",
"order_by": "timestamp",
"limit": 5,
"columns": ["timestamp"],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert data["start"] == "2024-01-01 00:00:00"
assert data["end"] == "2024-01-02 03:00:00"
def test_time_column_none_no_time_filter() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"columns": ["timestamp", "event"],
"time_column": "",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert "start" not in data and "end" not in data
assert len(data["rows"]) == 4
def test_test_dataset_int32_time_s() -> None:
app = server.create_app("TEST")
client = app.test_client()
payload = {
"table": "events",
"time_column": "ts",
"time_unit": "s",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert data["start"] == "2024-01-01 00:00:00"
assert data["end"] == "2024-01-01 01:00:00"
assert len(data["rows"]) == 2
def test_test_dataset_int32_time_us() -> None:
app = server.create_app("TEST")
client = app.test_client()
payload = {
"table": "events",
"time_column": "ts",
"time_unit": "us",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert data["start"] == "2024-01-01 00:00:00"
assert data["end"] == "2024-01-01 01:00:00"
assert len(data["rows"]) == 2I’ll take a look at the tests in test_server_timeseries.py now.
def test_group_by_table() -> None:
app = server.app
client = app.test_client()
payload: dict[str, Any] = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "table",
"order_by": "user",
"limit": 10,
"columns": ["value"],
"group_by": ["user"],
"aggregate": "Sum",
"show_hits": True,
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
rows = rv.get_json()["rows"]
assert rows[0][0] == "alice"
assert rows[0][1] == 2
assert rows[0][2] == 40
def test_table_avg_with_timestamp() -> None:
app = server.app
client = app.test_client()
payload: dict[str, Any] = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "table",
"order_by": "user",
"limit": 100,
"columns": ["user", "timestamp", "value"],
"group_by": ["user"],
"aggregate": "Avg",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert "error" not in data
rows = data["rows"]
assert rows[0][0] == "alice"
from dateutil import parser
ts = parser.parse(rows[0][2]).replace(tzinfo=None)
assert ts == parser.parse("2024-01-01 12:00:00")
def test_timeseries_basic() -> None:
app = server.app
client = app.test_client()
payload: dict[str, Any] = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "timeseries",
"limit": 100,
"group_by": ["user"],
"aggregate": "Count",
"columns": ["value"],
"x_axis": "timestamp",
"granularity": "1 day",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert len(data["rows"]) == 4
def test_timeseries_orders_by_xaxis() -> None:
app = server.app
client = app.test_client()
payload: dict[str, Any] = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "timeseries",
"limit": 100,
"columns": ["value"],
"x_axis": "timestamp",
"granularity": "1 day",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
rows = data["rows"]
from dateutil import parser
timestamps = [parser.parse(r[0]).replace(tzinfo=None) for r in rows]
assert timestamps == sorted(timestamps)
def test_timeseries_count_no_columns() -> None:
app = server.app
client = app.test_client()
payload: dict[str, Any] = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "timeseries",
"granularity": "1 day",
"columns": [],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
rows = data["rows"]
assert len(rows) == 2
assert rows[0][1] == 2
assert rows[1][1] == 2
assert len(rows[0]) == 3
def test_timeseries_limit_applies_to_series() -> None:
app = server.app
client = app.test_client()
payload: dict[str, Any] = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "timeseries",
"limit": 1,
"order_by": "user",
"group_by": ["user"],
"aggregate": "Count",
"columns": ["value"],
"x_axis": "timestamp",
"granularity": "1 day",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert len(data["rows"]) == 2
assert all(r[1] == "alice" for r in data["rows"])
def test_timeseries_auto_and_fine_buckets() -> None:
app = server.app
client = app.test_client()
def run(gran: str) -> None:
payload = {
"start": "2024-01-01 00:00:00",
"end": "2024-01-02 03:00:00",
"graph_type": "timeseries",
"columns": ["value"],
"x_axis": "timestamp",
"granularity": gran,
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/js
on"
)
data = rv.get_json()
assert rv.status_code == 200
from dateutil import parser
start = parser.parse(cast(str, payload["start"])).replace(tzinfo=None)
buckets = [
parser.parse(cast(str, r[0])).replace(tzinfo=None) for r in data["ro
ws"]
]
assert buckets[0] == start
if len(buckets) > 1:
step = (buckets[1] - buckets[0]).total_seconds()
assert step % data["bucket_size"] == 0
assert any(r[1] != 0 for r in data["rows"])
run("Auto")
run("Fine")
def test_timeseries_string_column_error() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "timeseries",
"limit": 7,
"columns": ["timestamp", "event", "value", "user"],I’m diving deeper into the code to look at 'group_by' for timeseries.
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
assert rv.status_code == 200
def test_derived_column_basic() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"limit": 10,
"columns": ["timestamp"],
"derived_columns": {"val2": "value * 2"},
"filters": [],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert data["rows"][0][1] == 20
def test_timeseries_derived_column() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "timeseries",
"granularity": "1 hour",
"limit": 7,
"columns": ["value"],
"derived_columns": {"derived_1": "value * 2"},
"aggregate": "Avg",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
rows = data["rows"]
assert all(r[3] == r[2] * 2 for r in rows)
def test_reserved_word_column() -> None:
app = server.create_app("TEST")
client = app.test_client()
payload = {
"table": "extra",
"columns": ["ts", "desc"],
"order_by": "ts",
"time_column": "",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert len(data["rows"]) == 2
assert data["rows"][0][1] == "x"
def test_order_by_samples_table() -> None:
app = server.app
client = app.test_client()
payload: dict[str, Any] = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "table",
"order_by": "Samples",
"order_dir": "DESC",
"limit": 10,
"columns": [],
"group_by": ["user"],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert 'ORDER BY "Hits" DESC' in data["sql"]
def test_order_by_samples_timeseries() -> None:
app = server.app
client = app.test_client()
payload: dict[str, Any] = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "timeseries",
"order_by": "Samples",
"order_dir": "DESC",
"x_axis": "timestamp",
"granularity": "1 day",
"columns": [],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert 'ORDER BY "Hits" DESC' in data["sql"]
def test_show_hits_client_side() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "table",
"columns": ["value"],
"group_by": ["user"],
"aggregate": "Sum",
"show_hits": False,
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert "count(*) AS Hits" in data["sql"]I’ll keep scrolling to see if there are additional tests to review.
root@3f0164aeb2f9:/workspace/scubaduck# sed -n '400,800p' tests/test_server_time
series.pyThe issue involves a mismatch between table headers and data. I'll check test_web_table.
tests/test_multi_table_web.py:29: select_value(page, "#table", "extra")
tests/test_multi_table_web.py:36: assert table_param == "extra"
tests/test_multi_table_web.py:44: select_value(page, "#table", "extra")
tests/test_multi_table_web.py:50: assert page.input_value("#table") == "extra
"
tests/test_multi_table_web.py:52: assert disp is not None and disp.strip() ==
"extra"
tests/test_multi_table_web.py:67: select_value(page, "#table", "extra")
tests/test_multi_table_web.py:68: page.wait_for_function("document.querySelec
tor('#table').value === 'extra'")
tests/test_server_timeseries.py:257: "table": "extra",I'll review the multi-table tests to ensure they're complete and accurate.
from typing import Any
import pytest
from werkzeug.serving import make_server
from scubaduck.server import create_app
from tests.web_utils import select_value
@pytest.fixture()
def multi_table_server_url() -> Iterator[str]:
app = create_app("TEST")
httpd = make_server("127.0.0.1", 0, app)
port = httpd.server_port
thread = threading.Thread(target=httpd.serve_forever)
thread.start()
try:
yield f"http://127.0.0.1:{port}"
finally:
httpd.shutdown()
thread.join()
def test_table_param_updates_on_dive(page: Any, multi_table_server_url: str) ->
None:
page.goto(multi_table_server_url + "?table=events")
page.wait_for_selector("#table option", state="attached")
select_value(page, "#table", "extra")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
table_param = page.evaluate(
"new URLSearchParams(window.location.search).get('table')"
)
assert table_param == "extra"
def test_table_dropdown_persists_on_refresh(
page: Any, multi_table_server_url: str
) -> None:
page.goto(multi_table_server_url + "?table=events")
page.wait_for_selector("#table option", state="attached")
select_value(page, "#table", "extra")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
page.reload()
page.wait_for_selector("#table option", state="attached")
assert page.input_value("#table") == "extra"
disp = page.text_content("#table + .dropdown-display")
assert disp is not None and disp.strip() == "extra"
def test_table_switch_resets_view_settings(
page: Any, multi_table_server_url: str
) -> None:
page.goto(multi_table_server_url + "?table=events")
page.wait_for_selector("#table option", state="attached")
page.click("text=Columns")
page.wait_for_selector("#column_groups input", state="attached")
page.uncheck("#column_groups input:first-of-type")
page.click("text=View Settings")
select_value(page, "#graph_type", "table")
page.fill("#limit", "50")
page.evaluate("g => { groupBy.chips = ['name']; groupBy.renderChips(); }")
select_value(page, "#table", "extra")
page.wait_for_function("document.querySelector('#table').value === 'extra'")
assert page.input_value("#graph_type") == "samples"
assert page.input_value("#limit") == "100"
chips = page.evaluate("groupBy.chips.length")
assert chips == 0
page.click("text=Columns")
page.wait_for_selector("#column_groups input", state="attached")
count = page.evaluate(
"document.querySelectorAll('#column_groups input:checked').length"
)
assert count == 3root@3f0164aeb2f9:/workspace/scubaduck# sed -n '80,160p' tests/test_multi_table_
web.pyThe table view data doesn't align with headers. I'll check tests/test_web_table.py for more details.
from tests.web_utils import run_query, select_value
@pytest.fixture()
def test_dataset_server_url() -> Iterator[str]:
app = create_app("TEST")
httpd = make_server("127.0.0.1", 0, app)
port = httpd.server_port
thread = threading.Thread(target=httpd.serve_forever)
thread.start()
try:
yield f"http://127.0.0.1:{port}"
finally:
httpd.shutdown()
thread.join()
def test_table_sorting(page: Any, server_url: str) -> None:
run_query(
page,
server_url,
start="2024-01-01 00:00:00",
end="2024-01-03 00:00:00",
order_by="user",
order_dir="ASC",
limit=100,
)
# header alignment
align = page.evaluate(
"getComputedStyle(document.querySelector('#results th')).textAlign"
)
assert align == "left"
header = page.locator("#results th").nth(3)
def values() -> list[str]:
return page.locator("#results td:nth-child(4)").all_inner_texts()
orig_rows = values()
assert orig_rows == ["alice", "bob", "alice", "charlie"]
first_sql = page.evaluate("window.lastResults.sql")
header.click()
assert values() == sorted(orig_rows)
assert header.inner_text().endswith("▲")
color = page.evaluate(
"getComputedStyle(document.querySelector('#results th:nth-child(4)')).co
lor"
)
assert "0, 0, 255" in color
assert page.evaluate("window.lastResults.sql") == first_sql
header.click()
assert values() == sorted(orig_rows, reverse=True)
assert header.inner_text().endswith("▼")
header.click()
assert values() == orig_rows
assert header.inner_text() == "user"
color = page.evaluate(
"getComputedStyle(document.querySelector('#results th:nth-child(4)')).co
lor"
)
assert "0, 0, 255" not in color
def test_table_avg_group_by(page: Any, server_url: str) -> None:
data = run_query(
page,
server_url,
start="2024-01-01 00:00:00",
end="2024-01-03 00:00:00",
order_by="timestamp",
group_by=["user"],
aggregate="Avg",
)
assert "error" not in data
assert len(data["rows"]) == 3
def test_table_enhancements(page: Any, server_url: str) -> None:
run_query(
page,
server_url,
start="2024-01-01 00:00:00",
end="2024-01-03 00:00:00",
order_by="timestamp",
limit=10,
)
border = page.evaluate(
"getComputedStyle(document.querySelector('#results td')).borderStyle"
)
assert border == "solid"
color1 = page.evaluate(
"getComputedStyle(document.querySelector('#results tr:nth-child(2) td'))
.backgroundColor"
)
color2 = page.evaluate(
"getComputedStyle(document.querySelector('#results tr:nth-child(3) td'))
.backgroundColor"
)
assert color1 != color2
page.hover("#results tr:nth-child(2)")
hover_color = page.evaluate(
"getComputedStyle(document.querySelector('#results tr:nth-child(2) td'))
.backgroundColor"
)
assert hover_color != color1
page.click("#results tr:nth-child(2)")
selected_color = page.evaluate(
"getComputedStyle(document.querySelector('#results tr:nth-child(2) td'))
.backgroundColor"
)
assert "189, 228, 255" in selected_color
overflow = page.evaluate(
"var v=document.getElementById('view'); v.scrollWidth > v.clientWidth"
)
assert not overflow
def test_table_single_selection(page: Any, server_url: str) -> None:
run_query(
page,
server_url,
start="2024-01-01 00:00:00",
end="2024-01-03 00:00:00",
order_by="timestamp",
limit=10,
)
page.click("#results tr:nth-child(2)")
page.click("#results tr:nth-child(3)")
count = page.evaluate("document.querySelectorAll('#results tr.selected').len
gth")
assert count == 1
is_third = page.evaluate(
"document.querySelector('#results tr:nth-child(3)').classList.contains('
selected')"
)
assert is_third
def test_timestamp_rendering(page: Any, server_url: str) -> None:
run_query(
page,
server_url,
start="2024-01-01 00:00:00",
end="2024-01-02 00:00:00",
order_by="timestamp",
limit=1,
)
cell = page.text_content("#results td")
assert cell != "Invalid Date"
valid = page.evaluate("v => !isNaN(Date.parse(v))", cell)
assert valid
def test_empty_data_message(page: Any, server_url: str) -> None:
data = run_query(
page,
server_url,
start="2025-01-01 00:00:00",
end="2025-01-02 00:00:00",
order_by="timestamp",
limit=100,
)
assert data["rows"] == []
msg = page.text_content("#view")
assert "Empty data provided to table" in msg
def test_group_by_chip_from_url(page: Any, server_url: str) -> None:
url = f"{server_url}?graph_type=table&group_by=user&order_by=user&limit=10"
page.goto(url)
page.wait_for_selector("#group_by_field .chip", state="attached")
chips = page.evaluate(
"Array.from(document.querySelectorAll('#group_by_field .chip')).map(c =>
c.firstChild.textContent)"
)
assert chips == ["user"]
def test_group_by_autocomplete(page: Any, server_url: str) -> None:
page.goto(f"{server_url}?graph_type=table")
page.wait_for_selector("#group_by_field", state="visible")
inp = page.query_selector("#group_by_field .f-val")
assert inp
inp.click()
page.keyboard.type("us")
page.wait_for_selector("#group_by_field .chip-dropdown div")
options = page.locator("#group_by_field .chip-dropdown div").all_inner_texts
()
assert "user" in options
def test_group_by_copy_icon(page: Any, server_url: str) -> None:
page.goto(f"{server_url}?graph_type=table")
page.wait_for_selector("#group_by_field", state="visible")
icon = page.text_content("#group_by_field .chip-copy")
assert icon == "⎘"
def test_group_by_input_no_border(page: Any, server_url: str) -> None:
page.goto(f"{server_url}?graph_type=table")
page.wait_for_selector("#group_by_field", state="visible")
border = page.evaluate(
"getComputedStyle(document.querySelector('#group_by_field .f-val')).bord
erStyle"
)
assert border == "none"
def test_table_group_by_query(page: Any, server_url: str) -> None:
data = run_query(
page,
server_url,
start="2024-01-01 00:00:00",
end="2024-01-03 00:00:00",
order_by="user",
limit=100,
group_by=["user"],
aggregate="Count",
)
assert "error" not in data
assert len(data["rows"]) == 3
def test_table_avg_no_group_by(page: Any, server_url: str) -> None:
data = run_query(
page,
server_url,
aggregate="Avg",
order_by="timestamp",
)
assert len(data["rows"]) == 1
row = data["rows"][0]I'll continue scrolling past line 240 to gather more details.
def test_table_headers_show_aggregate(page: Any, server_url: str) -> None:
run_query(
page,
server_url,
aggregate="Avg",
order_by="timestamp",
)
headers = page.locator("#results th").all_inner_texts()
assert "Hits" in headers
assert "timestamp (avg)" in headers
assert "value (avg)" in headers
def test_format_number_function(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
vals = page.evaluate(
"() => [formatNumber(815210), formatNumber(999.999), formatNumber(0.0004
), formatNumber(0)]"
)
assert vals == ["815.21 K", "999.999", "0.000", "0"]
def test_numeric_cell_nowrap(page: Any, server_url: str) -> None:
run_query(page, server_url, order_by="timestamp", limit=10)
whitespace = page.evaluate(
"getComputedStyle(document.querySelector('#results td:nth-child(3)')).wh
iteSpace"
)
assert whitespace == "nowrap"
def test_date_cell_nowrap(page: Any, server_url: str) -> None:
run_query(page, server_url, order_by="timestamp", limit=10)
whitespace = page.evaluate(
"getComputedStyle(document.querySelector('#results td:nth-child(1)')).wh
iteSpace"
)
assert whitespace == "nowrap"
def test_derived_column_query(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
select_value(page, "#order_by", "timestamp")
page.click("text=Columns")
page.click("text=Add Derived")
expr = page.query_selector("#derived_list .derived textarea")
assert expr
expr.fill("value * 2")
page.click("text=View Settings")
page.fill("#start", "2024-01-01 00:00:00")
page.fill("#end", "2024-01-03 00:00:00")
page.fill("#limit", "10")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
data = page.evaluate("window.lastResults")
assert data["rows"][0][-1] == 20
def test_derived_column_remove(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
page.click("text=Columns")
page.click("text=Add Derived")
assert page.query_selector("#derived_list .derived button.remove")
page.click("#derived_list .derived button.remove")
count = page.evaluate("document.querySelectorAll('#derived_list .derived').l
ength")
assert count == 0
def test_sql_query_display(page: Any, server_url: str) -> None:
data = run_query(
page,
server_url,
start="2024-01-01 00:00:00",
end="2024-01-02 00:00:00",
order_by="timestamp",
limit=10,
)
sql = data["sql"]
displayed = page.text_content("#sql_query")
assert displayed is not None
assert displayed.strip() == sql
def test_table_count_no_columns(page: Any, test_dataset_server_url: str) -> None
:
page.goto(test_dataset_server_url)
page.wait_for_selector("#order_by option", state="attached")
select_value(page, "#graph_type", "table")
page.click("text=Columns")
page.click("#columns_all")
page.click("text=View Settings")
page.evaluate("groupBy.chips = ['id']; groupBy.renderChips();")
select_value(page, "#aggregate", "Count")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
headers = page.locator("#results th").all_inner_texts()
assert headers == ["id", "Hits"]
col_count = page.locator("#results th").count()
row_count = page.locator("#results tr").count()
assert col_count == 2
assert row_count == 3
overflow = page.evaluate(
"var v=document.getElementById('view'); v.scrollWidth > v.clientWidth"
)
assert not overflowI’m passing over additional lines to explore the group_by section and possible extra tables.
root@3f0164aeb2f9:/workspace/scubaduck# sed -n '480,960p' tests/test_web_table.p
yI'll check the tests/test_web_graph file for any timeseries, starting from line 0.
root@3f0164aeb2f9:/workspace/scubaduck# ls tests | grep web
test_multi_table_web.py
test_web.py
test_web_basics.py
test_web_chips.py
test_web_columns.py
test_web_graph.py
test_web_table.py
test_web_timeseries.py
web_utils.pyI'll open tests/web_utils.py to check out how run_query is defined.
from typing import Any
def select_value(page: Any, selector: str, value: str) -> None:
page.evaluate(
"arg => setSelectValue(arg.sel, arg.val)",
{"sel": selector, "val": value},
)
def run_query(
page: Any,
url: str,
*,
start: str | None = None,
end: str | None = None,
order_by: str | None = None,
order_dir: str | None = "ASC",
limit: int | None = None,
group_by: list[str] | None = None,
aggregate: str | None = None,
) -> dict[str, Any]:
page.goto(url)
page.wait_for_selector("#order_by option", state="attached")
page.wait_for_selector("#order_dir", state="attached")
page.wait_for_function("window.lastResults !== undefined")
if start is not None:
page.fill("#start", start)
if end is not None:
page.fill("#end", end)
if order_by is not None:
select_value(page, "#order_by", order_by)
if order_dir is not None and order_dir == "DESC":
page.click("#order_dir")
if limit is not None:
page.fill("#limit", str(limit))
if group_by is not None:
select_value(page, "#graph_type", "table")
page.evaluate(
"g => { groupBy.chips = g; groupBy.renderChips(); }",
group_by,
)
if aggregate is not None:
select_value(page, "#graph_type", "table")
select_value(page, "#aggregate", aggregate)
if page.input_value("#graph_type") != "samples":
page.click("text=Columns")
page.wait_for_selector("#column_groups input", state="attached")
if not page.is_checked("#column_groups input[value='value']"):
page.check("#column_groups input[value='value']")
order_col = order_by or page.input_value("#order_by")
if order_col and not page.is_checked(
f"#column_groups input[value='{order_col}']"
):
if page.query_selector(f"#column_groups input[value='{order_col}']")
:
page.check(f"#column_groups input[value='{order_col}']")
page.click("text=View Settings")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
return page.evaluate("window.lastResults")Trying to fix misalignment issue when aggregating by 'num' in the TEST dataset.
root@3f0164aeb2f9:/workspace/scubaduck# grep -n "extra" -n tests/test_server_tim
eseries.py
257: "table": "extra",I'll check those specific lines closely to see what's going on.
root@3f0164aeb2f9:/workspace/scubaduck# sed -n '248,270p' tests/test_server_time
series.py
assert rv.status_code == 200
rows = data["rows"]
assert all(r[3] == r[2] * 2 for r in rows)
def test_reserved_word_column() -> None:
app = server.create_app("TEST")
client = app.test_client()
payload = {
"table": "extra",
"columns": ["ts", "desc"],
"order_by": "ts",
"time_column": "",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert len(data["rows"]) == 2
assert data["rows"][0][1] == "x"I’ll check tests/test_web_table for extra tables and search for 'test_dataset_server_url'.
_table.py
16:def test_dataset_server_url() -> Iterator[str]:
333:def test_table_count_no_columns(page: Any, test_dataset_server_url: str) ->
None:
334: page.goto(test_dataset_server_url)I’ll search for "extra" in test files to investigate if there are relevant tests.
root@3f0164aeb2f9:/workspace/scubaduck# grep -n "extra" -n tests/test_web_table.
pyI'll look for additional tests to explore.
root@3f0164aeb2f9:/workspace/scubaduck# grep -n "reserved_word" -r tests
tests/test_server_timeseries.py:253:def test_reserved_word_column() -> None:The bug is likely from improper column handling when grouping by 'num' and aggregating with count.
root@3f0164aeb2f9:/workspace/scubaduck# grep -n "show_hits" -n scubaduck/server.
py
46: show_hits: bool = False
510: show_hits=payload.get("show_hits", False),
535: params.group_by or params.aggregate or params.show_hits
540: "error": "group_by, aggregate and show_hits are only
valid for table or timeseries view"I'll search for 'Hits' to confirm if there's filtering with show_hits set to false.
307: select_parts.insert(len(group_cols), "count(*) AS Hits")
308: selected_for_order.add("Hits")
315: order_by = "Hits"
519: params.order_by = "Hits"
548: valid_cols.add("Hits")Let me review the code around line 519 to understand it better.
root@3f0164aeb2f9:/workspace/scubaduck# sed -n '512,562p' scubaduck/server.py
granularity=payload.get("granularity", "Auto"),
fill=payload.get("fill", "0"),
table=payload.get("table", default_table),
time_column=payload.get("time_column", "timestamp"),
time_unit=payload.get("time_unit", "s"),
)
if params.order_by == "Samples":
params.order_by = "Hits"
for f in payload.get("filters", []):
params.filters.append(Filter(f["column"], f["op"], f.get("value")))
if params.table not in tables:
return jsonify({"error": "Invalid table"}), 400
column_types = get_columns(params.table)
if params.time_column and params.time_column not in column_types:
return jsonify({"error": "Invalid time_column"}), 400
if params.time_unit not in {"s", "ms", "us", "ns"}:
return jsonify({"error": "Invalid time_unit"}), 400
if params.graph_type not in {"table", "timeseries"} and (
params.group_by or params.aggregate or params.show_hits
):
return (
jsonify(
{
"error": "group_by, aggregate and show_hits are only val
id for table or timeseries view"
}
),
400,
)
valid_cols = set(column_types.keys())
valid_cols.update(params.derived_columns.keys())
valid_cols.add("Hits")
if params.graph_type == "timeseries":
if params.x_axis is None:
params.x_axis = params.time_column
if params.x_axis is None or params.x_axis not in valid_cols:
return jsonify({"error": "Invalid x_axis"}), 400
ctype = column_types.get(params.x_axis, "").upper()
is_time = any(t in ctype for t in ["TIMESTAMP", "DATE", "TIME"])
is_numeric = any(
t in ctype
for t in [
"INT",
"DECIMAL",
"REAL",
"DOUBLE",I'm looking back around line 315 for further details.
root@3f0164aeb2f9:/workspace/scubaduck# sed -n '300,318p' scubaduck/server.py
selected_for_order.add("Count")
else:
for col in params.columns:
if col in group_cols:
continue
select_parts.append(f"{agg_expr(col)} AS {_quote(col)}")
selected_for_order.add(col)
select_parts.insert(len(group_cols), "count(*) AS Hits")
selected_for_order.add("Hits")
else:
select_parts.extend(_quote(c) for c in params.columns)
selected_for_order.update(params.columns)
order_by = params.order_by
if order_by == "Samples":
order_by = "Hits"
order_by = order_by if order_by in selected_for_order else None
if has_agg and params.derived_columns:I'm rethinking how the query should show results depending on the aggregator used. Let's check the tests for test_table_count.
root@3f0164aeb2f9:/workspace/scubaduck# grep -n "table_count" -r tests | head -n
20
tests/test_web_table.py:333:def test_table_count_no_columns(page: Any, test_data
set_server_url: str) -> None:I saw the test earlier with the aggregator count, looks like "Hits" is the only column shown.
y
page,
server_url,
start="2024-01-01 00:00:00",
end="2024-01-02 00:00:00",
order_by="timestamp",
limit=10,
)
sql = data["sql"]
displayed = page.text_content("#sql_query")
assert displayed is not None
assert displayed.strip() == sql
def test_table_count_no_columns(page: Any, test_dataset_server_url: str) -> None
:
page.goto(test_dataset_server_url)
page.wait_for_selector("#order_by option", state="attached")
select_value(page, "#graph_type", "table")
page.click("text=Columns")
page.click("#columns_all")
page.click("text=View Settings")
page.evaluate("groupBy.chips = ['id']; groupBy.renderChips();")
select_value(page, "#aggregate", "Count")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
headers = page.locator("#results th").all_inner_texts()
assert headers == ["id", "Hits"]
col_count = page.locator("#results th").count()
row_count = page.locator("#results tr").count()
assert col_count == 2
assert row_count == 3
overflow = page.evaluate(
"var v=document.getElementById('view'); v.scrollWidth > v.clientWidth"
)
assert not overflowThe query appends "count(*) AS Count" for aggregation but only returns "id, Hits" in tests.
run_query automatically selects the "value" column, but manual calls might override that.
root@3f0164aeb2f9:/workspace/scubaduck# ls scubaduck/static/js
chip_input.js table_view.js timeseries_chart.js
sidebar.js time_column.js view_settings.jsTime to open up table_view.js and dig into what’s going on there!
root@3f0164aeb2f9:/workspace/scubaduck# sed -n '1,200p' scubaduck/static/js/tabl
e_view.js
// Logic for rendering the table based views. Extracted from index.html so that
// the inline script only handles wiring up the UI.
let originalRows = [];
let sortState = { index: null, dir: null };
function renderTable(rows) {
const table = document.getElementById("results");
table.innerHTML = "";
if (rows.length === 0) return;
let hitsIndex = selectedColumns.indexOf("Hits");
let totalHits = 0;
if (hitsIndex !== -1) {
totalHits = rows.reduce((s, r) => s + Number(r[hitsIndex]), 0);
}
const header = document.createElement("tr");
selectedColumns.forEach((col, i) => {
const th = document.createElement("th");
let label = col;
if (
displayType === "table" &&
col !== "Hits" &&
!(groupBy.chips || []).includes(col)
) {
const agg = document.getElementById("aggregate").value.toLowerCase();
label += ` (${agg})`;
}
th.textContent = label;
th.dataset.index = i;
th.addEventListener("click", handleSort);
if (sortState.index === i) {
th.classList.add("sorted");
th.textContent = label + (sortState.dir === "desc" ? " \u25BC" : " \u25B2"
);
}
th.style.textAlign = "left";
header.appendChild(th);
});
table.appendChild(header);
rows.forEach((row) => {
const tr = document.createElement("tr");
tr.addEventListener("click", () => {
const wasSelected = tr.classList.contains("selected");
document
.querySelectorAll("#results tr.selected")
.forEach((el) => el.classList.remove("selected"));
if (!wasSelected) {
tr.classList.add("selected");
}
});
row.forEach((v, i) => {
const col = selectedColumns[i];
const td = document.createElement("td");
if (isTimeColumn(col)) {
let d;
const t = (columnTypes[col] || "").toUpperCase();
if (t.includes("TIMESTAMP") || t.includes("DATE") || t.includes("TIME"))
{
d = new Date(v);
} else {
const unit = document.getElementById("time_unit").value;
const factors = { s: 1000, ms: 1, us: 0.001, ns: 0.000001 };
d = new Date(Number(v) * (factors[unit] || 1000));
}
td.textContent = d.toLocaleString("en-US", {
weekday: "short",
month: "short",
day: "numeric",
year: "numeric",
hour: "numeric",
minute: "numeric",
second: "numeric",
hour12: true,
timeZoneName: "short",
});
td.classList.add("date");
} else {
if (col === "Hits") {
const pct = totalHits ? ((v / totalHits) * 100).toFixed(1) : "0";
td.textContent = `${formatNumber(v)} (${pct}%)`;
} else {
td.textContent = isStringColumn(col) ? v : formatNumber(v);
}
}
if (!isStringColumn(col) && !isTimeColumn(col)) {
td.classList.add("numeric");
}
td.style.textAlign = isStringColumn(col) ? "left" : "right";
tr.appendChild(td);
});
table.appendChild(tr);
});
// ensure table does not overflow unless necessary
const view = document.getElementById("view");
if (table.scrollWidth <= view.clientWidth) {
table.style.width = "100%";
}
}
function handleSort(e) {
const idx = parseInt(e.target.dataset.index, 10);
if (sortState.index !== idx) {
sortState.index = idx;
sortState.dir = "asc";
} else if (sortState.dir === "asc") {
sortState.dir = "desc";
} else if (sortState.dir === "desc") {
sortState.index = null;
sortState.dir = null;
} else {
sortState.dir = "asc";
}
let rows = originalRows.slice();
if (sortState.index !== null) {
rows.sort((a, b) => {
const va = a[sortState.index];
const vb = b[sortState.index];
if (va === vb) return 0;
if (sortState.dir === "desc") return va < vb ? 1 : -1;
return va > vb ? 1 : -1;
});
}
renderTable(rows);
}
function showResults(data) {
window.lastResults = data;
const hideHits =
(graphTypeSel.value === "table" || graphTypeSel.value === "timeseries") &&
!document.getElementById("show_hits").checked;
if (hideHits && data.rows.length) {
const groupCount =
(graphTypeSel.value === "timeseries" ? 1 : 0) +
((groupBy.chips || []).length || 0);
data.rows.forEach((r) => r.splice(groupCount, 1));
}
const view = document.getElementById("view");
if (graphTypeSel.value === "timeseries") {
showTimeSeries(data);
} else {
if (data.rows.length === 0) {
view.innerHTML =
'<p id="empty-message">Empty data provided to table</p><table id="result
s"></table>';
} else {
view.innerHTML = '<table id="results"></table>';
}
originalRows = data.rows.slice();
sortState = { index: null, dir: null };
renderTable(originalRows);
}
const sqlEl = document.createElement("pre");
sqlEl.id = "sql_query";
sqlEl.style.whiteSpace = "pre-wrap";
sqlEl.style.marginTop = "10px";
sqlEl.textContent = data.sql;
view.appendChild(sqlEl);
document.getElementById("query_info").textContent = `Your query took about ${l
astQueryTime} ms`;
}
function showError(err) {
window.lastResults = err;
const view = document.getElementById("view");
let msg = "";
if (typeof err === "string") {
msg = err;
} else if (err) {
msg = err.error || "Error";
if (err.sql) {
msg += "\nSQL: " + err.sql;
}
if (err.traceback) {
msg += "\n" + err.traceback;
}
}
view.innerHTML = `<pre id="error-message">${msg}</pre>`;
document.getElementById("query_info").textContent = "";
}The showResults function removes the Hits column if unchecked. I should check selectedColumns earlier.
root@3f0164aeb2f9:/workspace/scubaduck# sed -n '1,200p' scubaduck/static/js/side
bar.js
// Sidebar resizing logic extracted from index.html
const sidebar = document.getElementById('sidebar');
const sidebarResizer = document.getElementById('sidebar-resizer');
let sidebarWidth = parseInt(localStorage.getItem('sidebarWidth') || 450, 10);
sidebar.style.width = sidebarWidth + 'px';
let sidebarResize = false;
function startSidebarDrag(e) {
e.preventDefault();
sidebarResize = true;
document.addEventListener('mousemove', onSidebarDrag);
document.addEventListener('mouseup', stopSidebarDrag);
}
function onSidebarDrag(e) {
if (!sidebarResize) return;
sidebarWidth = Math.max(200, e.clientX - sidebar.getBoundingClientRect().left)
;
sidebar.style.width = sidebarWidth + 'px';
}
function stopSidebarDrag() {
document.removeEventListener('mousemove', onSidebarDrag);
document.removeEventListener('mouseup', stopSidebarDrag);
sidebarResize = false;
localStorage.setItem('sidebarWidth', sidebarWidth);
}
sidebarResizer.addEventListener('mousedown', startSidebarDrag);I’m going to open view_settings.js to work on building the query.
root@3f0164aeb2f9:/workspace/scubaduck# sed -n '1,200p' scubaduck/static/js/view
_settings.js
// Logic for View Settings, Columns, and URL handling extracted from index.html
const allColumns = [];
const baseColumns = [];
const columnTypes = {};
const stringColumns = [];
const baseStringColumns = [];
const integerColumns = [];
const baseIntegerColumns = [];
const timeColumns = [];
const baseTimeColumns = [];
const timeColumnOptions = [];
const baseTimeColumnOptions = [];
const derivedColumns = [];
let selectedColumns = [];
let displayType = 'samples';
let groupBy = {chips: [], addChip: () => {}, renderChips: () => {}};
let defaultTimeColumn = '';
const limitInput = document.getElementById('limit');
const defaultLimit = parseInt(limitInput.value, 10);
const limitValues = {
samples: defaultLimit,
table: defaultLimit,
timeseries: 7
};
const columnValues = {
samples: [],
table: [],
timeseries: []
};
limitInput.addEventListener('input', () => {
limitValues[displayType] = parseInt(limitInput.value, 10);
limitInput.dataset.setByUser = '1';
});
function initDropdown(select) {
// Avoid creating duplicate wrappers if this dropdown was already initialised.
if (select.dataset.dropdownInit) {
const disp = select.parentElement?.querySelector('.dropdown-display');
if (disp) {
const opt = select.options[select.selectedIndex];
disp.textContent = opt ? opt.textContent : '';
}
return;
}
select.dataset.dropdownInit = '1';
const wrapper = document.createElement('div');
wrapper.className = 'dropdown';
if (select.classList.contains('f-col')) {
wrapper.classList.add('f-col');
}
select.parentNode.insertBefore(wrapper, select);
wrapper.appendChild(select);
select.style.display = 'none';
const disp = document.createElement('div');
disp.className = 'dropdown-display';
function updateDisplay() {
const opt = select.options[select.selectedIndex];
disp.textContent = opt ? opt.textContent : '';
}
updateDisplay();
wrapper.appendChild(disp);
const menu = document.createElement('div');
menu.className = 'dropdown-menu';
const search = document.createElement('input');
search.placeholder = 'Search';
menu.appendChild(search);
const list = document.createElement('div');
menu.appendChild(list);
wrapper.appendChild(menu);
function close() {
menu.style.display = 'none';
}
function open() {
renderOptions();
menu.style.display = 'block';
search.focus();
}
disp.addEventListener('click', () => {
if (menu.style.display === 'block') {
close();
} else {
open();
}
});
document.addEventListener('click', e => {
if (!wrapper.contains(e.target)) {
close();
}
});
function renderOptions() {
const q = search.value.toLowerCase();
list.innerHTML = '';
Array.from(select.options).forEach(o => {
if (!o.textContent.toLowerCase().includes(q)) return;
const div = document.createElement('div');
div.className = 'option';
if (q) {
const text = o.textContent;
const idx = text.toLowerCase().indexOf(q);
if (idx !== -1) {
div.innerHTML =
text.slice(0, idx) +
'<u>' +
text.slice(idx, idx + q.length) +
'</u>' +
text.slice(idx + q.length);
} else {
div.textContent = text;
}
} else {
div.textContent = o.textContent;
}
if (o.value === select.value) div.classList.add('selected');
div.addEventListener('mousedown', evt => {
evt.preventDefault();
select.value = o.value;
select.dispatchEvent(new Event('change'));
updateDisplay();
close();
});
list.appendChild(div);
});
}
search.addEventListener('input', renderOptions);
select.addEventListener('change', updateDisplay);
}
let orderDir = 'ASC';
const orderDirBtn = document.getElementById('order_dir');
const graphTypeSel = document.getElementById('graph_type');
function updateOrderDirButton() {
orderDirBtn.textContent = orderDir + (orderDir === 'ASC' ? ' \u25B2' : ' \u25B
C');
}
function updateDisplayTypeUI() {
const prevType = displayType;
updateSelectedColumns(prevType);
const newType = graphTypeSel.value;
const showTable = newType === 'table';
const showTS = newType === 'timeseries';
document.getElementById('group_by_field').style.display = showTable || showTS
? 'flex' : 'none';
document.getElementById('aggregate_field').style.display = showTable || showTS
? 'flex' : 'none';
document.getElementById('show_hits_field').style.display = showTable ? 'flex'
: 'none';
document.getElementById('x_axis_field').style.display = showTS ? 'flex' : 'non
e';
document.getElementById('granularity_field').style.display = showTS ? 'flex' :
'none';
document.getElementById('fill_field').style.display = showTS ? 'flex' : 'none'
;
document.querySelectorAll('#column_groups .col-group').forEach(g => {
if (g.querySelector('.col-group-header').textContent.startsWith('Strings'))
{
g.style.display = showTable || showTS ? 'none' : '';
}
});
limitValues[prevType] = parseInt(limitInput.value, 10);
if (showTS && limitValues.timeseries === undefined) {
limitValues.timeseries = 7;
}
limitInput.value = limitValues[newType];
document.querySelectorAll('#column_groups input').forEach(cb => {
cb.checked = columnValues[newType].includes(cb.value);
});
if (showTS) {
document.querySelectorAll('#column_groups input').forEach(cb => {
if (isTimeColumn(cb.value) || isStringColumn(cb.value)) {
cb.checked = false;
}
});
document.getElementById('order_by').value = '';
}
updateSelectedColumns(newType);
displayType = newType;
}
function updateTimeFieldVisibility() {
const show = document.getElementById('time_column').value !== '';
document.getElementById('start').closest('.field').style.display = show
? 'flex'
: 'none';
document.getElementById('end').closest('.field').style.display = show
? 'flex'
: 'none';
document.getElementById('time_unit').style.display = show ? '' : 'none';
}
orderDirBtn.addEventListener('click', () => {
orderDir = orderDir === 'ASC' ? 'DESC' : 'ASC';
updateOrderDirButton();
});
updateOrderDirButton();
graphTypeSel.addEventListener('change', updateDisplayTypeUI);
document.getElementById('time_column').addEventListener('change', updateTimeFiel
dVisibility);
updateTimeFieldVisibility();
function loadColumns(table) {
return fetch('/api/columns?table=' + encodeURIComponent(table)).then(r => r.js
on()).then(cols => {
const orderSelect = document.getElementById('order_by');
const xAxisSelect = document.getElementById('x_axis');I'm searching for functions related to "selectedColumns" to understand their computation.
root@3f0164aeb2f9:/workspace/scubaduck# grep -n "selectedColumns" -n scubaduck/s
tatic/js/*
scubaduck/static/js/table_view.js:11: let hitsIndex = selectedColumns.indexOf("
Hits");
scubaduck/static/js/table_view.js:17: selectedColumns.forEach((col, i) => {
scubaduck/static/js/table_view.js:51: const col = selectedColumns[i];
scubaduck/static/js/timeseries_chart.js:38: const valueCols = selectedColumns.s
lice(groups.length + hasHits);
scubaduck/static/js/view_settings.js:15:let selectedColumns = [];
scubaduck/static/js/view_settings.js:447: selectedColumns = groupBy.chips.sli
ce();
scubaduck/static/js/view_settings.js:448: if (document.getElementById('show_h
its').checked) selectedColumns.push('Hits');
scubaduck/static/js/view_settings.js:452: if (!selectedColumns.includes(c
)) selectedColumns.push(c);
scubaduck/static/js/view_settings.js:455: if (dc.include && !selectedColu
mns.includes(dc.name)) selectedColumns.push(dc.name);
scubaduck/static/js/view_settings.js:459: selectedColumns = base.slice();
scubaduck/static/js/view_settings.js:461: if (dc.include) selectedColumns.p
ush(dc.name);
scubaduck/static/js/view_settings.js:464: columnValues[type] = selectedColumns.
slice();
scubaduck/static/js/view_settings.js:466: if (orderCol && !selectedColumns.incl
udes(orderCol)) {
scubaduck/static/js/view_settings.js:467: selectedColumns.push(orderCol);
scubaduck/static/js/view_settings.js:702: columns: selectedColumns.filter(c =
>I'll skip to around line 447 to check out the relevant section.
ew_settings.js
});
function updateColumnsTabCount() {
const baseCount = document.querySelectorAll('#column_groups input:checked').le
ngth;
const derivedCount = document.querySelectorAll('#derived_list .derived .d-use:
checked').length;
const btn = document.getElementById('columns_tab');
if (btn) btn.textContent = `Columns (${baseCount + derivedCount})`;
}
function updateSelectedColumns(type = graphTypeSel.value) {
const base = allColumns.filter(name => {
const cb = document.querySelector(`#column_groups input[value="${name}"]`);
if (!cb || !cb.checked) return false;
if (type === 'table' && isStringColumn(name)) return false;
return true;
});
if (type === 'table' || type === 'timeseries') {
selectedColumns = groupBy.chips.slice();
if (document.getElementById('show_hits').checked) selectedColumns.push('Hits
');
const agg = document.getElementById('aggregate').value.toLowerCase();
if (!(type === 'table' && agg === 'count')) {
base.forEach(c => {
if (!selectedColumns.includes(c)) selectedColumns.push(c);
});
derivedColumns.forEach(dc => {
if (dc.include && !selectedColumns.includes(dc.name)) selectedColumns.pu
sh(dc.name);
});
}
} else {
selectedColumns = base.slice();
derivedColumns.forEach(dc => {
if (dc.include) selectedColumns.push(dc.name);
});
}
columnValues[type] = selectedColumns.slice();
const orderCol = document.getElementById('order_by').value;
if (orderCol && !selectedColumns.includes(orderCol)) {
selectedColumns.push(orderCol);
}
updateColumnsTabCount();
}
function isStringColumn(name) {
const t = (columnTypes[name] || '').toUpperCase();
return t.includes('CHAR') || t.includes('STRING') || t.includes('VARCHAR');
}
function isIntegerColumn(name) {
const t = (columnTypes[name] || '').toUpperCase();
return t.includes('INT');
}
function isTimeColumn(name) {
const t = (columnTypes[name] || '').toUpperCase();
if (t.includes('TIMESTAMP') || t.includes('DATE') || t.includes('TIME')) retur
n true;
const sel = document.getElementById('time_column').value;
const xsel = document.getElementById('x_axis').value;
if (name === sel || name === xsel) return true;
return false;
}
function formatNumber(val) {
if (typeof val !== 'number') val = Number(val);
if (Number.isNaN(val)) return '';
if (val === 0) return '0';
const abs = Math.abs(val);
if (abs > 999.999) {
const units = [
{n: 1e12, s: 'T'},
{n: 1e9, s: 'B'},
{n: 1e6, s: 'M'},
{n: 1e3, s: 'K'},
];
for (const u of units) {
if (abs >= u.n) {
return (val / u.n).toFixed(2) + ' ' + u.s;
}
}
}
if (abs < 0.0005) return '0.000';
if (Number.isInteger(val)) return val.toString();
return val.toFixed(3);
}
function addFilter() {
const container = document.createElement('div');
container.className = 'filter';
container.innerHTML = `
<div class="filter-row">
<select class="f-col"></select>
<select class="f-op"></select>
<button type="button" class="remove" onclick="this.closest('.filter').remo
ve()">✖</button>
</div>
<div class="chip-box">
<div class="chip-input">
<input class="f-val" type="text">
<button type="button" class="chip-copy">⎘</button>
</div>
<div class="chip-dropdown"></div>
</div>
`;
const colSel = container.querySelector('.f-col');
colSel.innerHTML = allColumns.map(c => `<option value="${c}">${c}</option>`).j
oin('');
initDropdown(colSel);
function populateOps() {
const opSel = container.querySelector('.f-op');
const col = colSel.value;
const ops = isStringColumn(col)
? [
['=', '='],
['!=', '!='],
['~', 'matches regex'],
['!~', 'not matches regex'],
['contains', 'contains'],
...
>
<textarea class="d-expr" rows="2"></textarea>
`;
document.getElementById('derived_list').appendChild(container);
const obj = {
type: data.type || 'string',
name: data.name || nextDerivedName(),
expr: data.expr || '',
include: data.include !== undefined ? data.include : true,
el: container
};
container.querySelector('.d-type').value = obj.type;
container.querySelector('.d-name').value = obj.name;
container.querySelector('.d-expr').value = obj.expr;
container.querySelector('.d-use').checked = obj.include;
['change','input'].forEach(evt => {
container.addEventListener(evt, refreshDerivedColumns);
});
derivedColumns.push(obj);
refreshDerivedColumns();
}
function removeDerived(btn) {
const el = btn.closest('.derived');
const idx = derivedColumns.findIndex(d => d.el === el);
if (idx !== -1) {
derivedColumns.splice(idx, 1);
}
el.remove();
refreshDerivedColumns();
}
function refreshDerivedColumns() {
allColumns.splice(0, allColumns.length, ...baseColumns);
stringColumns.splice(0, stringColumns.length, ...baseStringColumns);
integerColumns.splice(0, integerColumns.length, ...baseIntegerColumns);
timeColumns.splice(0, timeColumns.length, ...baseTimeColumns);
timeColumnOptions.splice(0, timeColumnOptions.length, ...baseTimeColumnOptions
);
derivedColumns.forEach(d => {
d.type = d.el.querySelector('.d-type').value;
d.name = d.el.querySelector('.d-name').value;
d.expr = d.el.querySelector('.d-expr').value;
d.include = d.el.querySelector('.d-use').checked;
allColumns.push(d.name);
columnTypes[d.name] = d.type;
if (d.type === 'string') {
stringColumns.push(d.name);
} else {
integerColumns.push(d.name);
timeColumnOptions.push(d.name);
}
});
updateSelectedColumns();
}
let lastQueryTime = 0;
let queryStart = 0;
function dive(push=true) {
const params = collectParams();
if (push) {
history.pushState(params, '', paramsToSearch(params));
}
const payload = Object.assign({}, params);
const dcMap = {};
(params.derived_columns || []).forEach(d => {
if (d.include) dcMap[d.name] = d.expr;
});
payload.derived_columns = dcMap;
const view = document.getElementById('view');
view.innerHTML = '<p>Loading...</p>';
window.lastResults = undefined;
queryStart = performance.now();
fetch('/api/query', {method:'POST', headers:{'Content-Type':'application/json'
}, body:JSON.stringify(payload)})
.then(async r => {
const data = await r.json();
if (!r.ok) throw data;
return data;
})
.then(data => {
lastQueryTime = Math.round(performance.now() - queryStart);
showResults(data);
})
.catch(err => {
showError(err);
});
}
function collectParams() {
updateSelectedColumns();
const payload = {
table: document.getElementById('table').value,
time_column: document.getElementById('time_column').value,
time_unit: document.getElementById('time_unit').value,
start: document.getElementById('start').value,
end: document.getElementById('end').value,
order_by: document.getElementById('order_by').value,
order_dir: orderDir,
limit: parseInt(document.getElementById('limit').value, 10),
columns: selectedColumns.filter(c =>
c !== 'Hits' && !derivedColumns.some(dc => dc.name === c)
),
samples_columns: columnValues.samples.slice(),
table_columns: columnValues.table.slice(),
timeseries_columns: columnValues.timeseries.slice(),
graph_type: graphTypeSel.value,
filters: Array.from(document.querySelectorAll('#filters .filter')).map(f =>
{
const chips = f.chips || [];
const op = f.querySelector('.f-op').value;
let value = null;
if (op !== 'empty' && op !== '!empty') {
value = chips.length === 0 ? null : (chips.length === 1 ? chips[0] : chi
ps);
}
const colSel = f.querySelector('.f-col select') || f.querySelector('.f-col
');
return {column: colSel.value, op, value};
}),
derived_columns: Array.from(document.querySelectorAll('#derived_list .derive
d')).map(d => ({
type: d.querySelector('.d-type').value,I’m continuing to scroll to line 702 to check the details.
root@3f0164aeb2f9:/workspace/scubaduck# sed -n '720,768p' scubaduck/static/js/vi
ew_settings.js
type: d.querySelector('.d-type').value,
name: d.querySelector('.d-name').value,
expr: d.querySelector('.d-expr').value,
include: d.querySelector('.d-use').checked,
}))
};
if (graphTypeSel.value === 'table' || graphTypeSel.value === 'timeseries') {
payload.group_by = groupBy.chips || [];
payload.aggregate = document.getElementById('aggregate').value;
payload.show_hits = document.getElementById('show_hits').checked;
}
if (graphTypeSel.value === 'timeseries') {
const xval = document.getElementById('x_axis').value;
if (xval) payload.x_axis = xval;
payload.granularity = document.getElementById('granularity').value;
payload.fill = document.getElementById('fill').value;
}
return payload;
}
function paramsToSearch(params) {
const sp = new URLSearchParams();
if (params.table) sp.set('table', params.table);
if (params.time_column) sp.set('time_column', params.time_column);
if (params.time_unit) sp.set('time_unit', params.time_unit);
if (params.start) sp.set('start', params.start);
if (params.end) sp.set('end', params.end);
if (params.order_by) sp.set('order_by', params.order_by);
if (params.order_dir) sp.set('order_dir', params.order_dir);
if (params.limit !== null && params.limit !== undefined) sp.set('limit', param
s.limit);
if (params.samples_columns && params.samples_columns.length) sp.set('samples_c
olumns', params.samples_columns.join(','));
if (params.table_columns && params.table_columns.length) sp.set('table_columns
', params.table_columns.join(','));
if (params.timeseries_columns && params.timeseries_columns.length) sp.set('tim
eseries_columns', params.timeseries_columns.join(','));
if (params.filters && params.filters.length) sp.set('filters', JSON.stringify(
params.filters));
if (params.derived_columns && params.derived_columns.length) sp.set('derived_c
olumns', JSON.stringify(params.derived_columns));
if (params.graph_type) sp.set('graph_type', params.graph_type);
if (params.graph_type === 'table' || params.graph_type === 'timeseries') {
if (params.group_by && params.group_by.length) sp.set('group_by', params.gro
up_by.join(','));
if (params.aggregate) sp.set('aggregate', params.aggregate);
if (params.show_hits) sp.set('show_hits', '1');
}
if (params.graph_type === 'timeseries') {
if (params.x_axis) sp.set('x_axis', params.x_axis);
if (params.granularity) sp.set('granularity', params.granularity);
if (params.fill) sp.set('fill', params.fill);
}
const qs = sp.toString();
return qs ? '?' + qs : '';
}In updateSelectedColumns, base columns are excluded for the 'count' aggregator when the type is 'table'. The 'Hits' column only appears in selectedColumns if show_hits is checked. When show_hits is false, it won't appear in selectedColumns, but showResults will hide it from rows if needed.
There’s an extra column when "group_by num" and "aggregate count" are used. I'll check if 'show_hits' is unchecked by default in index.html.
root@3f0164aeb2f9:/workspace/scubaduck# grep -n "show_hits" -n -r scubaduck/stat
ic
scubaduck/static/index.html:324: <div id="show_hits_field" class="field"
style="display:none;">
scubaduck/static/index.html:326: <input id="show_hits" type="checkbox"
checked>
scubaduck/static/js/timeseries_chart.js:32: const hasHits = document.getElement
ById('show_hits').checked ? 1 : 0;
scubaduck/static/js/view_settings.js:150: document.getElementById('show_hits_fi
eld').style.display = showTable ? 'flex' : 'none';
scubaduck/static/js/view_settings.js:448: if (document.getElementById('show_h
its').checked) selectedColumns.push('Hits');
scubaduck/static/js/view_settings.js:729: payload.show_hits = document.getEle
mentById('show_hits').checked;
scubaduck/static/js/view_settings.js:759: if (params.show_hits) sp.set('show_
hits', '1');
scubaduck/static/js/view_settings.js:802: document.getElementById('show_hits').
checked = params.show_hits ?? true;
scubaduck/static/js/view_settings.js:849: document.getElementById('show_hits').
checked = true;
scubaduck/static/js/view_settings.js:892: if (sp.has('show_hits')) params.show_
hits = sp.get('show_hits') === '1';
scubaduck/static/js/table_view.js:128: !document.getElementById("show_hits").
checked;It looks like index.html sets the show_hits checkbox to checked by default, so it's true.
root@3f0164aeb2f9:/workspace/scubaduck# sed -n '320,336p' scubaduck/static/index
.html
<option>p99.9</option>
<option>p99.99</option>
</select>
</div>
<div id="show_hits_field" class="field" style="display:none;">
<label>Show Hits</label>
<input id="show_hits" type="checkbox" checked>
</div>
<div id="filters">
<h4>Filters<span class="help" title="You can create as many filters as
you want. You can either write a filter using a UI or manual SQL. In the UI, fi
lter consists of a column name, a relation (e.g., =, !=, <, >) and then a text f
ield. The text field is a token input. It accepts multiple tokens for = relation
, in which case we match using an OR for all options.">[?]</span></h4>
<div id="filter_list"></div>
<button id="add_filter" type="button" onclick="addFilter()">Add Filter
</button>
</div>
<div id="query_info" style="margin-top:10px;"></div>
</div>
<div id="columns" class="tab-content">
<div id="column_actions">When show_hits is checked, 'Hits' gets added, affecting query params and server response.
I'll investigate if isTimeColumn handles undefined values correctly to avoid errors.
root@3f0164aeb2f9:/workspace/scubaduck# grep -n "function isTimeColumn" -n scuba
duck/static/js/view_settings.js
482:function isTimeColumn(name) {root@3f0164aeb2f9:/workspace/scubaduck# sed -n '480,506p' scubaduck/static/js/vi
ew_settings.js
}
function isTimeColumn(name) {
const t = (columnTypes[name] || '').toUpperCase();
if (t.includes('TIMESTAMP') || t.includes('DATE') || t.includes('TIME')) retur
n true;
const sel = document.getElementById('time_column').value;
const xsel = document.getElementById('x_axis').value;
if (name === sel || name === xsel) return true;
return false;
}
function formatNumber(val) {
if (typeof val !== 'number') val = Number(val);
if (Number.isNaN(val)) return '';
if (val === 0) return '0';
const abs = Math.abs(val);
if (abs > 999.999) {
const units = [
{n: 1e12, s: 'T'},
{n: 1e9, s: 'B'},
{n: 1e6, s: 'M'},
{n: 1e3, s: 'K'},
];
for (const u of units) {
if (abs >= u.n) {
return (val / u.n).toFixed(2) + ' ' + u.s;
}It looks like isTimeColumn won't throw an error since it falls back to an empty string.
The issue lies in build_query adding "Count" when it shouldn't, leading to column misalignment.
root@3f0164aeb2f9:/workspace/scubaduck# grep -n "Count" -r tests | head -n 20
tests/test_server_errors.py:59: "aggregate": "Count",
tests/test_web_table.py:226: aggregate="Count",
tests/test_web_table.py:341: select_value(page, "#aggregate", "Count")
tests/test_server_time.py:67: "aggregate": "Count",
tests/test_server_timeseries.py:71: "aggregate": "Count",
tests/test_server_timeseries.py:143: "aggregate": "Count",I'm reviewing lines around test_web_table and tests/test_server_timeseries for the aggregator count.
root@3f0164aeb2f9:/workspace/scubaduck# sed -n '60,80p' tests/test_server_timese
ries.py
def test_timeseries_basic() -> None:
app = server.app
client = app.test_client()
payload: dict[str, Any] = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "timeseries",
"limit": 100,
"group_by": ["user"],
"aggregate": "Count",
"columns": ["value"],
"x_axis": "timestamp",
"granularity": "1 day",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200I'm checking the next 80 lines to ensure I have the full information.
eries.py
assert rv.status_code == 200
assert len(data["rows"]) == 4
def test_timeseries_orders_by_xaxis() -> None:
app = server.app
client = app.test_client()
payload: dict[str, Any] = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "timeseries",
"limit": 100,
"columns": ["value"],
"x_axis": "timestamp",
"granularity": "1 day",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
rows = data["rows"]
from dateutil import parser
timestamps = [parser.parse(r[0]).replace(tzinfo=None) for r in rows]
assert timestamps == sorted(timestamps)
def test_timeseries_count_no_columns() -> None:
app = server.app
client = app.test_client()
payload: dict[str, Any] = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "timeseries",
"granularity": "1 day",
"columns": [],
}
rv = client.post(I'm continuing to scroll down to capture the next set of lines.
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
rows = data["rows"]
assert len(rows) == 2
assert rows[0][1] == 2
assert rows[1][1] == 2
assert len(rows[0]) == 3
def test_timeseries_limit_applies_to_series() -> None:
app = server.app
client = app.test_client()
payload: dict[str, Any] = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "timeseries",
"limit": 1,
"order_by": "user",
"group_by": ["user"],
"aggregate": "Count",
"columns": ["value"],
"x_axis": "timestamp",
"granularity": "1 day",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert len(data["rows"]) == 2
assert all(r[1] == "alice" for r in data["rows"])
def test_timeseries_auto_and_fine_buckets() -> None:
app = server.app
client = app.test_client()The test expects 3 columns: bucket, Hits, and one more. I’ll check further context.
series.py
def test_timeseries_count_no_columns() -> None:
app = server.app
client = app.test_client()
payload: dict[str, Any] = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "timeseries",
"granularity": "1 day",
"columns": [],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
rows = data["rows"]
assert len(rows) == 2
assert rows[0][1] == 2
assert rows[1][1] == 2
assert len(rows[0]) == 3
def test_timeseries_limit_applies_to_series() -> None:
app = server.app
client = app.test_client()
payload: dict[str, Any] = {
"table": "events",Looking at the timeseries test, it expects three columns: bucket, Hits, and Count, but columns are empty. The server returns two columns because of the default aggregate "count". The test only expects the row length to be 3, without verifying column names.
I’m investigating if 'Count' should be removed to match the expected format in the tests.
root@3f0164aeb2f9:/workspace/scubaduck# grep -n "len(rows[0])" -r tests/test_ser
ver_timeseries.pyI'll check the test_web_timeseries.py file to verify if there’s anything else.
s.py
57: assert path_zero is not None and path_zero.count("L") > 20
64: assert path_conn is not None and path_conn.count("M") == 1
71: assert path_blank is not None and path_blank.count("M") > 1
120: count = page.eval_on_selector_all("#crosshair_dots circle", "els => els.
length")
121: assert count > 0
220: count = page.eval_on_selector_all("#chart path", "els => els.length")
221: assert count == 2
289: count = page.eval_on_selector_all("#chart text.tick-label", "els => els.
length")
290: assert count > 2
304: count = page.eval_on_selector_all("#chart text.y-tick-label", "els => el
s.length")
305: grid_count = page.eval_on_selector_all("#chart line.grid", "els => els.l
ength")
306: assert count > 0 and count == grid_countI'm skimming through about 350 lines of the file to check the features.
from tests.web_utils import select_value
def test_timeseries_default_query(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#graph_type", state="attached")
select_value(page, "#graph_type", "timeseries")
page.click("text=Columns")
page.check("#column_groups input[value='value']")
page.click("text=View Settings")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
data = page.evaluate("window.lastResults")
assert "error" not in data
assert page.is_visible("#chart")
page.click("text=Columns")
assert not page.is_checked("#column_groups input[value='timestamp']")
def test_timeseries_single_bucket(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#graph_type", state="attached")
page.fill("#start", "2024-01-01 00:00:00")
page.fill("#end", "2024-01-01 00:00:00")
select_value(page, "#graph_type", "timeseries")
page.click("text=Columns")
page.check("#column_groups input[value='value']")
page.click("text=View Settings")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
path = page.get_attribute("#chart path", "d")
assert path is not None and "NaN" not in path
def test_timeseries_fill_options(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#graph_type", state="attached")
page.fill("#start", "2024-01-01 00:00:00")
page.fill("#end", "2024-01-02 03:00:00")
select_value(page, "#graph_type", "timeseries")
page.click("text=Columns")
page.check("#column_groups input[value='value']")
page.click("text=View Settings")
select_value(page, "#granularity", "1 hour")
select_value(page, "#fill", "0")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
path_zero = page.get_attribute("#chart path", "d")
assert path_zero is not None and path_zero.count("L") > 20
select_value(page, "#fill", "connect")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
path_conn = page.get_attribute("#chart path", "d")
assert path_conn is not None and path_conn.count("M") == 1
select_value(page, "#fill", "blank")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
path_blank = page.get_attribute("#chart path", "d")
assert path_blank is not None and path_blank.count("M") > 1
def test_timeseries_hover_highlight(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#graph_type", state="attached")
select_value(page, "#graph_type", "timeseries")
page.click("text=Columns")
page.check("#column_groups input[value='value']")
page.click("text=View Settings")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
page.wait_for_selector("#chart path", state="attached")
path_el = page.query_selector("#chart path")
assert path_el
page.evaluate(
"el => el.dispatchEvent(new MouseEvent('mouseenter', {bubbles: true}))",
path_el,
)
width = page.evaluate(
"getComputedStyle(document.querySelector('#chart path')).strokeWidth"
)
assert "2.5" in width
color = page.evaluate(
"getComputedStyle(document.querySelector('#legend .legend-item')).backgr
oundColor"
)
assert "221, 221, 221" in color
def test_timeseries_crosshair(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#graph_type", state="attached")
select_value(page, "#graph_type", "timeseries")
page.click("text=Columns")
page.check("#column_groups input[value='value']")
page.click("text=View Settings")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
page.wait_for_selector("#chart path", state="attached")
page.eval_on_selector(
"#chart",
"el => { const r = el.getBoundingClientRect(); el.dispatchEvent(new Mous
eEvent('mousemove', {clientX: r.left + r.width/2, clientY: r.top + r.height/2, b
ubbles: true})); }",
)
line_display = page.evaluate(
"document.getElementById('crosshair_line').style.display"
)
assert line_display != "none"
count = page.eval_on_selector_all("#crosshair_dots circle", "els => els.leng
th")I'm checking the bottom of the file to find more tests.
root@3f0164aeb2f9:/workspace/scubaduck# grep -n "no_columns" -n tests/test_web_t
imeseries.pyI'm looking into timeseries_count_no_columns test and considering how the UI behaves with counts.
root@3f0164aeb2f9:/workspace/scubaduck# sed -n '1,160p' scubaduck/static/js/time
series_chart.js
let resizeObserver = null;
let currentChart = null;
function showTimeSeries(data) {
function parseTs(s) {
if (s.match(/GMT/) || s.endsWith('Z') || /\+\d{2}:?\d{2}$/.test(s)) {
return new Date(s).getTime();
}
return new Date(s + 'Z').getTime();
}
const view = document.getElementById('view');
if (data.rows.length === 0) {
view.innerHTML = '<p id="empty-message">Empty data provided to table</p>';
return;
}
const height = 600;
view.innerHTML =
'<div id="ts-container"><div id="legend"></div><div id="chart-wrapper"><svg
id="chart" height="' +
height +
'"></svg></div></div>';
const svg = document.getElementById('chart');
const legend = document.getElementById('legend');
const crosshairLine = document.createElementNS('http://www.w3.org/2000/svg', '
line');
crosshairLine.id = 'crosshair_line';
crosshairLine.setAttribute('stroke', '#555');
crosshairLine.style.display = 'none';
const crosshairDots = document.createElementNS('http://www.w3.org/2000/svg', '
g');
crosshairDots.id = 'crosshair_dots';
crosshairDots.style.display = 'none';
const groups = groupBy.chips || [];
const hasHits = document.getElementById('show_hits').checked ? 1 : 0;
const fill = document.getElementById('fill').value;
const bucketMs = (data.bucket_size || 3600) * 1000;
const start = data.start ? parseTs(data.start) : null;
const end = data.end ? parseTs(data.end) : null;
const startIdx = 1 + groups.length + hasHits;
const valueCols = selectedColumns.slice(groups.length + hasHits);
const series = {};
data.rows.forEach(r => {
const ts = parseTs(r[0]);
const groupKey = groups.map((_, i) => r[1 + i]).join(':') || 'all';
valueCols.forEach((name, i) => {
const val = Number(r[startIdx + i]);
const key = groupKey === 'all' ? name : groupKey + ':' + name;
if (!series[key]) series[key] = {};
series[key][ts] = val;
});
});
const buckets = [];
let minX = start !== null ? start : Infinity;
let maxX = end !== null ? end : -Infinity;
if (start !== null && end !== null) {
for (let t = start; t <= end; t += bucketMs) {
buckets.push(t);
}
} else {
Object.keys(series).forEach(k => {
const s = series[k];
Object.keys(s).forEach(t => {
const n = Number(t);
if (n < minX) minX = n;
if (n > maxX) maxX = n;
});
});
for (let t = minX; t <= maxX; t += bucketMs) {
buckets.push(t);
}
}
let minY = Infinity,
maxY = -Infinity;
Object.keys(series).forEach(key => {
const vals = series[key];
buckets.forEach(b => {
const v = vals[b];
const val = v === undefined && fill === '0' ? 0 : v;
if (val === undefined) return;
if (val < minY) minY = val;
if (val > maxY) maxY = val;
});
});
if (fill === '0') {
if (minY > 0) minY = 0;
if (maxY < 0) maxY = 0;
}
const colors = [
'#1f77b4',
'#ff7f0e',
'#2ca02c',
'#d62728',
'#9467bd',
'#8c564b',
'#e377c2'
];
currentChart = {
svg,
legend,
series,
buckets,
minX,
maxX,
minY,
maxY,
fill,
colors,
height,
crosshairLine,
crosshairDots,
seriesEls: {},
bucketPixels: [],
xScale: null,
yScale: null,
selected: null,
frozen: false
};
const intervals = [
{unit: 'second', step: 1, ms: 1000},
{unit: 'second', step: 2, ms: 2000},
{unit: 'second', step: 5, ms: 5000},
{unit: 'second', step: 10, ms: 10000},
{unit: 'second', step: 15, ms: 15000},
{unit: 'second', step: 30, ms: 30000},
{unit: 'minute', step: 1, ms: 60000},
{unit: 'minute', step: 2, ms: 120000},
{unit: 'minute', step: 5, ms: 300000},
{unit: 'minute', step: 10, ms: 600000},
{unit: 'minute', step: 15, ms: 900000},
{unit: 'minute', step: 30, ms: 1800000},
{unit: 'hour', step: 1, ms: 3600000},
{unit: 'hour', step: 2, ms: 7200000},
{unit: 'hour', step: 3, ms: 10800000},
{unit: 'hour', step: 4, ms: 14400000},
{unit: 'hour', step: 6, ms: 21600000},
{unit: 'hour', step: 12, ms: 43200000},
{unit: 'day', step: 1, ms: 86400000},
{unit: 'day', step: 2, ms: 172800000},
{unit: 'week', step: 1, ms: 604800000},
{unit: 'week', step: 2, ms: 1209600000},
{unit: 'month', step: 1},
{unit: 'month', step: 3},
{unit: 'month', step: 6},
{unit: 'year', step: 1},
{unit: 'year', step: 2},
{unit: 'year', step: 5},
{unit: 'year', step: 10}
];
function chooseInterval(start, end) {
const span = end - start;
function approxMs(i) {
if (i.ms) return i.ms;
if (i.unit === 'month') return i.step * 2629800000;
if (i.unit === 'year') return i.step * 31557600000;
return 1000;
}For timeseries, selectedColumns are set based on group and hit options, and bucket index is used.
In table view with count aggregator, avoid including "count(*) AS Count" in query.
When columns are selected, show "Count" for each column in table view, not omit it.
I need to check tests for aggregate "Count" with selected columns and compare.
root@3f0164aeb2f9:/workspace/scubaduck# grep -n "aggregate\": \"Count\"" -r test
s | head -n 20
tests/test_server_errors.py:59: "aggregate": "Count",
tests/test_server_time.py:67: "aggregate": "Count",
tests/test_server_timeseries.py:71: "aggregate": "Count",
tests/test_server_timeseries.py:143: "aggregate": "Count",I'll check tests/test_server_time.py, specifically the part involving the aggregator Count.
root@3f0164aeb2f9:/workspace/scubaduck# sed -n '60,80p' tests/test_server_time.p
y
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-01 01:00:00",
"graph_type": "timeseries",
"granularity": "1 hour",
"columns": ["event"],
"aggregate": "Count",
"time_column": "created",
"time_unit": "ms",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert len(data["rows"]) == 2
def test_integer_time_unit_us_default_start_end(tmp_path: Path) -> None:
csv_file = tmp_path / "events.csv"It appears the code is returning extra columns when using count aggregation with "event."
Timeseries isn't showing "event" correctly due to a mismatch with aggregator count and selectedColumns.
Need to modify query not to include "Count" column for table views with aggregator count.
I'll create a test for server queries with aggregate count and show_hits for 'extra'.
I'll check how the server handles time_column="" and ensure the correct SQL query is generated.
I’ll check for tests in test_server_basic or test_server_timeseries, using "extra" as a search term.
root@3f0164aeb2f9:/workspace/scubaduck# grep -n "extra" -r tests | head -n 20
tests/test_multi_table_web.py:29: select_value(page, "#table", "extra")
tests/test_multi_table_web.py:36: assert table_param == "extra"
tests/test_multi_table_web.py:44: select_value(page, "#table", "extra")
tests/test_multi_table_web.py:50: assert page.input_value("#table") == "extra
"
tests/test_multi_table_web.py:52: assert disp is not None and disp.strip() ==
"extra"
tests/test_multi_table_web.py:67: select_value(page, "#table", "extra")
tests/test_multi_table_web.py:68: page.wait_for_function("document.querySelec
tor('#table').value === 'extra'")
tests/test_server_timeseries.py:257: "table": "extra",I'm debating whether to add the test in test_server_basic or test_server_timeseries. Let’s check the style of test_server_timeseries.
py
app = server.app
client = app.test_client()
payload: dict[str, Any] = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "timeseries",
"order_by": "Samples",
"order_dir": "DESC",
"x_axis": "timestamp",
"granularity": "1 day",
"columns": [],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert 'ORDER BY "Hits" DESC' in data["sql"]
def test_show_hits_client_side() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "table",
"columns": ["value"],
"group_by": ["user"],
"aggregate": "Sum",
"show_hits": False,
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert "count(*) AS Hits" in data["sql"]I'll examine the earlier tests near test_reserved_word_column to match the style.
"filters": [],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert data["rows"][0][1] == 20
def test_timeseries_derived_column() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "timeseries",
"granularity": "1 hour",
"limit": 7,
"columns": ["value"],
"derived_columns": {"derived_1": "value * 2"},
"aggregate": "Avg",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
rows = data["rows"]
assert all(r[3] == r[2] * 2 for r in rows)
def test_reserved_word_column() -> None:
app = server.create_app("TEST")
client = app.test_client()
payload = {
"table": "extra",
"columns": ["ts", "desc"],
"order_by": "ts",
"time_column": "",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert len(data["rows"]) == 2
assert data["rows"][0][1] == "x"I'll add a test for count with group_by, sorting rows, and verifying no "Count" column.
I’m opening codex_setup.sh to ensure the .venv environment is properly set up.
uv sync --frozen
source .venv/bin/activate
python -c "import os; import duckdb; con = duckdb.connect(); con.execute(f\"SET
http_proxy = '{os.getenv(\"HTTP_PROXY\")}'\"); con.execute(\"INSTALL 'sqlite';\"
)"
playwright install chromium
echo "source .venv/bin/activate" >> ~/.bashrcI need to activate the virtual environment by running source .venv/bin/activate.
root@3f0164aeb2f9:/workspace/scubaduck# source .venv/bin/activate
(scubaduck) root@3f0164aeb2f9:/workspace/scubaduck#I'll start by running targeted server tests, then run full checks after the patch.
(scubaduck) root@3f0164aeb2f9:/workspace/scubaduck# git status --shortserver.py
f"TIMESTAMP '{params.start}' + INTERVAL '{sec} second' * "
f"CAST(floor((epoch({xexpr}) - epoch(TIMESTAMP '{params.start}')
)/{sec}) AS BIGINT)"
)
else:
bucket_expr = (
f"TIMESTAMP 'epoch' + INTERVAL '{sec} second' * "
f"CAST(floor(epoch({xexpr})/{sec}) AS BIGINT)"
)
select_parts.append(f"{bucket_expr} AS bucket")
group_cols = ["bucket"] + group_cols
selected_for_order.add("bucket")
has_agg = bool(group_cols) or params.aggregate is not None
if has_agg:
select_cols = (
group_cols[1:] if params.graph_type == "timeseries" else group_cols
)
select_parts.extend(_quote(c) for c in select_cols)
agg = (params.aggregate or "count").lower()
selected_for_order.update(group_cols)
def agg_expr(col: str) -> str:
expr = _quote(col)
ctype = column_types.get(col, "").upper() if column_types else ""
if "BOOL" in ctype:
expr = f"CAST({_quote(col)} AS BIGINT)"
if agg.startswith("p"):
quant = float(agg[1:]) / 100
return f"quantile({expr}, {quant})"
if agg == "count distinct":
return f"count(DISTINCT {expr})"
if agg == "avg" and column_types is not None:
if "TIMESTAMP" in ctype or "DATE" in ctype or "TIME" in ctype:
return (
"TIMESTAMP 'epoch' + INTERVAL '1 second' * "
f"CAST(avg(epoch({_quote(col)})) AS BIGINT)"
)
return f"{agg}({expr})"
if agg == "count":
select_parts.append("count(*) AS Count")
selected_for_order.add("Count")
else:
for col in params.columns:
if col in group_cols:
continue
select_parts.append(f"{agg_expr(col)} AS {_quote(col)}")
selected_for_order.add(col)
select_parts.insert(len(group_cols), "count(*) AS Hits")
selected_for_order.add("Hits")
else:
select_parts.extend(_quote(c) for c in params.columns)
selected_for_order.update(params.columns)
order_by = params.order_by
if order_by == "Samples":
(scubaduck) root@3f0164aeb2f9:/workspace/scubaduck#Time to make some tweaks and apply a fix to this part.
(scubaduck) root@3f0164aeb2f9:/workspace/scubaduck# apply_patch<<'PATCH'
> *** Begin Patch
> *** Update File: scubaduck/server.py
> @@
> - if agg == "count":
> - select_parts.append("count(*) AS Count")
> - selected_for_order.add("Count")
> + if agg == "count":
> + if params.graph_type != "table":
> + select_parts.append("count(*) AS Count")
> + selected_for_order.add("Count")
> *** End Patch
> PATCH
Success. Updated the following files:
M scubaduck/server.py
(scubaduck) root@3f0164aeb2f9:/workspace/scubaduck#I'll add a new server test, maybe after the "test_reserved_word_column" one.
(scubaduck) root@3f0164aeb2f9:/workspace/scubaduck# tail -n 20 tests/test_server
_timeseries.py
def test_show_hits_client_side() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "table",
"columns": ["value"],
"group_by": ["user"],
"aggregate": "Sum",
"show_hits": False,
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert "count(*) AS Hits" in data["sql"]
(scubaduck) root@3f0164aeb2f9:/workspace/scubaduck#I’ll add the new test either before or after, and patch as needed.
> *** Begin Patch
> *** Update File: tests/test_server_timeseries.py
> @@
> def test_reserved_word_column() -> None:
> @@
> assert len(data["rows"]) == 2
> assert data["rows"][0][1] == "x"
> +
> +
> +def test_table_count_group_by_extra() -> None:
> + app = server.create_app("TEST")
> + client = app.test_client()
> + payload = {
> + "table": "extra",
> + "graph_type": "table",
> + "time_column": "",
> + "group_by": ["num"],
> + "aggregate": "Count",
> + "columns": [],
> + }
> + rv = client.post(
> + "/api/query", data=json.dumps(payload), content_type="application/jso
n"
> + )
> + data = rv.get_json()
> + assert rv.status_code == 200
> + rows = sorted(data["rows"])
> + assert rows == [[1, 1], [2, 1]]
> + assert all(len(r) == 2 for r in rows)
> *** End Patch
> PATCH
Success. Updated the following files:
M tests/test_server_timeseries.py(scubaduck) root@3f0164aeb2f9:/workspace/scubaduck# ruff format scubaduck/server
.py tests/test_server_timeseries.py
2 files left unchanged(scubaduck) root@3f0164aeb2f9:/workspace/scubaduck# ruff check scubaduck/server.
py tests/test_server_timeseries.py
All checks passed!(scubaduck) root@3f0164aeb2f9:/workspace/scubaduck# pyright/workspace/scubaduck/tests/test_server_errors.py
/workspace/scubaduck/tests/test_server_errors.py:65:5 - error: Variable "data"
is not accessed (reportUnusedVariable)
1 error, 0 warnings, 0 informations
(scubaduck) root@3f0164aeb2f9:/workspace/scubaduck#I see an error at line 65 in tests/test_server_errors.py. I'll check the file.
(scubaduck) root@3f0164aeb2f9:/workspace/scubaduck# sed -n '60,70p' tests/test_s
erver_errors.py
"show_hits": True,
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
def test_samples_view_rejects_group_by() -> None:
app = server.apperver_errors.py
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "table",
"order_by": "timestamp",
"limit": 100,
"columns": ["user", "Hits", "value"],
"group_by": ["user"],
"aggregate": "Count",
"show_hits": True,
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
def test_samples_view_rejects_group_by() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"graph_type": "samples",
"group_by": ["user"],
"columns": ["timestamp"],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
(scubaduck) root@3f0164aeb2f9:/workspace/scubaduck#I'll open more lines to check the rest since the path got truncated.
erver_errors.py
"show_hits": True,
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
def test_samples_view_rejects_group_by() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"graph_type": "samples",
"group_by": ["user"],
"columns": ["timestamp"],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 400
assert "only valid" in data["error"]
(scubaduck) root@3f0164aeb2f9:/workspace/scubaduck#Pyright is warning about 'data' being unused, even though get_json() is called. Let's check for config changes.
[project]
name = "scubaduck"
version = "0.1.0"
description = "Scuba on DuckDB"
readme = "README.md"
requires-python = ">=3.12"
license = {text = "MIT"}
dependencies = [
"duckdb",
"flask",
"python-dateutil>=2.9.0.post0",
]
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"
[tool.pytest.ini_options]
addopts = "-n auto --tb=native"
[tool.pyright]
# Pyright configuration with strict settings
include = ["scubaduck", "tests"]
exclude = ["**/__pycache__", "dist"]
stubPath = "stubs"
venvPath = "."
venv = ".venv"
reportMissingImports = true
reportMissingTypeStubs = true
pythonVersion = "3.12"
pythonPlatform = "All"
typeCheckingMode = "strict"
reportUnknownMemberType = true
reportUnknownParameterType = true
reportUnknownVariableType = true
reportUnknownArgumentType = true
reportPrivateImportUsage = true
reportUntypedFunctionDecorator = true
reportFunctionMemberAccess = true
reportIncompatibleMethodOverride = true
[tool.ruff]
target-version = "py312"
[tool.uv]
dev-dependencies = [
"pytest>=8.3.5",
"ruff>=0.11.2",
"pytest-xdist>=3.6.1",
"playwright>=1.52.0",
"pyright>=1.1.350",
"pytest-playwright>=0.7.0",
]
(scubaduck) root@3f0164aeb2f9:/workspace/scubaduck#I'm reviewing the line numbers and the config to fix any unused variable errors.
(scubaduck) root@3f0164aeb2f9:/workspace/scubaduck# nl -ba tests/test_server_err
ors.py | sed -n '55,75p'
55 "order_by": "timestamp",
56 "limit": 100,
57 "columns": ["user", "Hits", "value"],
58 "group_by": ["user"],
59 "aggregate": "Count",
60 "show_hits": True,
61 }
62 rv = client.post(
63 "/api/query", data=json.dumps(payload), content_type="applicatio
n/json"
64 )
65 data = rv.get_json()
66 assert rv.status_code == 200
67
68
69 def test_samples_view_rejects_group_by() -> None:
70 app = server.app
71 client = app.test_client()
72 payload = {
73 "table": "events",
74 "graph_type": "samples",
75 "group_by": ["user"],
(scubaduck) root@3f0164aeb2f9:/workspace/scubaduck#The variable is defined but not used. I'll check lines around 60 for potential fixes.
(scubaduck) root@3f0164aeb2f9:/workspace/scubaduck# sed -n '50,70p' tests/test_s
erver_errors.py
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "table",
"order_by": "timestamp",
"limit": 100,
"columns": ["user", "Hits", "value"],
"group_by": ["user"],
"aggregate": "Count",
"show_hits": True,
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
def test_samples_view_rejects_group_by() -> None:
app = server.app
(scubaduck) root@3f0164aeb2f9:/workspace/scubaduck#I'll rename the variable on line 65 to _ = rv.get_json() or remove it.
(scubaduck) root@3f0164aeb2f9:/workspace/scubaduck# apply_patch<<'PATCH'
> *** Begin Patch
> *** Update File: tests/test_server_errors.py
> @@
> - rv = client.post(
> - "/api/query", data=json.dumps(payload), content_type="application/jso
n"
> - )
> - data = rv.get_json()
> - assert rv.status_code == 200
> + rv = client.post(
> + "/api/query", data=json.dumps(payload), content_type="application/jso
n"
> + )
> + _ = rv.get_json()
> + assert rv.status_code == 200
> *** End Patch
> PATCH
Success. Updated the following files:
M tests/test_server_errors.py
(scubaduck) root@3f0164aeb2f9:/workspace/scubaduck#I'll run ruff format and check the file once more for issues.
r_errors.py tests/test_server_timeseries.py scubaduck/server.py
3 files left unchanged(scubaduck) root@3f0164aeb2f9:/workspace/scubaduck# ruff check tests/test_server
_errors.py tests/test_server_timeseries.py scubaduck/server.py
All checks passed!
(scubaduck) root@3f0164aeb2f9:/workspace/scubaduck#I’ll give pyright another go to see if anything changes.
(scubaduck) root@3f0164aeb2f9:/workspace/scubaduck# pyright0 errors, 0 warnings, 0 informations
(scubaduck) root@3f0164aeb2f9:/workspace/scubaduck#Pyright passed! Now I'll go ahead and run pytest as instructed.
(scubaduck) root@3f0164aeb2f9:/workspace/scubaduck# pytest -qbringing up nodes........................................................................... [ 59%]................................................. [100%]
121 passed in 15.46s
(scubaduck) root@3f0164aeb2f9:/workspace/scubaduck#All tests have passed (121 tests). Time to commit the changes!
(scubaduck) root@3f0164aeb2f9:/workspace/scubaduck# git status --short
M scubaduck/server.py
M tests/test_server_errors.py
M tests/test_server_timeseries.py
(scubaduck) root@3f0164aeb2f9:/workspace/scubaduck#I'll commit with the message: "fix: table view count aggregator extra column."
(scubaduck) root@3f0164aeb2f9:/workspace/scubaduck# git add scubaduck/server.py
tests/test_server_errors.py tests/test_server_timeseries.py(scubaduck) root@3f0164aeb2f9:/workspace/scubaduck# git commit -m "fix(server):
drop duplicate count column in table view" && git status --short
[work 56f6016] fix(server): drop duplicate count column in table view
3 files changed, 25 insertions(+), 3 deletions(-)
(scubaduck) root@3f0164aeb2f9:/workspace/scubaduck#I’ll quickly confirm the current state of the commits.
(scubaduck) root@3f0164aeb2f9:/workspace/scubaduck# git status --short
(scubaduck) root@3f0164aeb2f9:/workspace/scubaduck#I'll summarize the changes and add relevant citation lines from server.py to the PR.
(scubaduck) root@3f0164aeb2f9:/workspace/scubaduck# nl -ba scubaduck/server.py |
sed -n '290,308p'
290 if agg == "avg" and column_types is not None:
291 if "TIMESTAMP" in ctype or "DATE" in ctype or "TIME" in
ctype:
292 return (
293 "TIMESTAMP 'epoch' + INTERVAL '1 second' * "
294 f"CAST(avg(epoch({_quote(col)})) AS BIGINT)"
295 )
296 return f"{agg}({expr})"
297
298 if agg == "count":
299 if params.graph_type != "table":
300 select_parts.append("count(*) AS Count")
301 selected_for_order.add("Count")
302 else:
303 for col in params.columns:
304 if col in group_cols:
305 continue
306 select_parts.append(f"{agg_expr(col)} AS {_quote(col)}")
307 selected_for_order.add(col)
308 select_parts.insert(len(group_cols), "count(*) AS Hits")
(scubaduck) root@3f0164aeb2f9:/workspace/scubaduck#I need to capture the lines around the new test in test_server_timeseries.py.
(scubaduck) root@3f0164aeb2f9:/workspace/scubaduck# nl -ba tests/test_server_tim
eseries.py | sed -n '252,288p'
252
253 def test_reserved_word_column() -> None:
254 app = server.create_app("TEST")
255 client = app.test_client()
256 payload = {
257 "table": "extra",
258 "columns": ["ts", "desc"],
259 "order_by": "ts",
260 "time_column": "",
261 }
262 rv = client.post(
263 "/api/query", data=json.dumps(payload), content_type="applicatio
n/json"
264 )
265 data = rv.get_json()
266 assert rv.status_code == 200
267 assert len(data["rows"]) == 2
268 assert data["rows"][0][1] == "x"
269
270
271 def test_table_count_group_by_extra() -> None:
272 app = server.create_app("TEST")
273 client = app.test_client()
274 payload = {
275 "table": "extra",
276 "graph_type": "table",
277 "time_column": "",
278 "group_by": ["num"],
279 "aggregate": "Count",
280 "columns": [],
281 }
282 rv = client.post(
283 "/api/query", data=json.dumps(payload), content_type="applicatio
n/json"
284 )
285 data = rv.get_json()
286 assert rv.status_code == 200
287 rows = sorted(data["rows"])
288 assert rows == [[1, 1], [2, 1]]
(scubaduck) root@3f0164aeb2f9:/workspace/scubaduck#I'll make sure to include the assert statement checking the row lengths are consistent.
(scubaduck) root@3f0164aeb2f9:/workspace/scubaduck# nl -ba tests/test_server_tim
eseries.py | sed -n '288,300p'
288 assert rows == [[1, 1], [2, 1]]
289 assert all(len(r) == 2 for r in rows)
290
291
292 def test_order_by_samples_table() -> None:
293 app = server.app
294 client = app.test_client()
295 payload: dict[str, Any] = {
296 "table": "events",
297 "start": "2024-01-01 00:00:00",
298 "end": "2024-01-03 00:00:00",
299 "graph_type": "table",
300 "order_by": "Samples",
(scubaduck) root@3f0164aeb2f9:/workspace/scubaduck#I’m grabbing the snippet from the modified line in test_server_errors for the _ variable.
(scubaduck) root@3f0164aeb2f9:/workspace/scubaduck# nl -ba tests/test_server_err
ors.py | sed -n '60,68p'
60 "show_hits": True,
61 }
62 rv = client.post(
63 "/api/query", data=json.dumps(payload), content_type="applicatio
n/json"
64 )
65 _ = rv.get_json()
66 assert rv.status_code == 200
67
68
(scubaduck) root@3f0164aeb2f9:/workspace/scubaduck#Time to create the commit message and put together the PR. Let's do this!