Task ID: task_e_682fdca66754832392977097843c80c4
Configuring language runtimes...
+ rm -- /tmp/3Q1XTL-setup_script.sh
+ source ./codex_setup.sh
++ uv sync --frozen
Using CPython 3.12.3 interpreter at: /usr/bin/python3.12
Creating virtual environment at: .venv
Building scubaduck @ file:///workspace/scubaduck
Downloading playwright (43.1MiB)
Downloading pyright (5.3MiB)
Downloading duckdb (19.3MiB)
Downloading ruff (11.0MiB)
Downloading ruff
Built scubaduck @ file:///workspace/scubaduck
Downloading duckdb
Downloading playwright
Downloading pyright
Prepared 33 packages in 1.55s
Installed 33 packages in 83ms
+ blinker==1.9.0
+ certifi==2025.4.26
+ charset-normalizer==3.4.2
+ click==8.2.0
+ duckdb==1.2.2
+ execnet==2.1.1
+ flask==3.1.1
+ greenlet==3.2.2
+ idna==3.10
+ iniconfig==2.1.0
+ itsdangerous==2.2.0
+ jinja2==3.1.6
+ markupsafe==3.0.2
+ nodeenv==1.9.1
+ packaging==25.0
+ playwright==1.52.0
+ pluggy==1.6.0
+ pyee==13.0.0
+ pyright==1.1.400
+ pytest==8.3.5
+ pytest-base-url==2.1.0
+ pytest-playwright==0.7.0
+ pytest-xdist==3.6.1
+ python-dateutil==2.9.0.post0
+ python-slugify==8.0.4
+ requests==2.32.3
+ ruff==0.11.10
+ scubaduck==0.1.0 (from file:///workspace/scubaduck)
+ six==1.17.0
+ text-unidecode==1.3
+ typing-extensions==4.13.2
+ urllib3==2.4.0
+ werkzeug==3.1.3
++ source .venv/bin/activate
+++ '[' -z '' ']'
+++ '[' -n x ']'
+++ SCRIPT_PATH=.venv/bin/activate
+++ '[' .venv/bin/activate = /tmp/3Q1XTL-setup_script.sh ']'
+++ deactivate nondestructive
+++ unset -f pydoc
+++ '[' -z '' ']'
+++ '[' -z '' ']'
+++ hash -r
+++ '[' -z '' ']'
+++ unset VIRTUAL_ENV
+++ unset VIRTUAL_ENV_PROMPT
+++ '[' '!' nondestructive = nondestructive ']'
+++ VIRTUAL_ENV=/workspace/scubaduck/.venv
+++ '[' linux-gnu = cygwin ']'
+++ '[' linux-gnu = msys ']'
+++ export VIRTUAL_ENV
+++ '[' -z '' ']'
+++ unset SCRIPT_PATH
+++ _OLD_VIRTUAL_PATH=/root/.cargo/bin:/root/.rbenv/shims:/root/.rbenv/bin:/root/.rbenv/shims:/root/.local/share/swiftly/bin:/root/.bun/bin:/root/.nvm/versions/node/v22.16.0/bin:/root/.pyenv/shims:3441PYENV_ROOT/shims:/root/.pyenv/bin:/usr/local/go/bin:/root/go/bin:/root/.rbenv/bin:/root/.rbenv/shims:/root/.bun/bin:/root/.local/bin:/root/.pyenv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+++ PATH=/workspace/scubaduck/.venv/bin:/root/.cargo/bin:/root/.rbenv/shims:/root/.rbenv/bin:/root/.rbenv/shims:/root/.local/share/swiftly/bin:/root/.bun/bin:/root/.nvm/versions/node/v22.16.0/bin:/root/.pyenv/shims:3441PYENV_ROOT/shims:/root/.pyenv/bin:/usr/local/go/bin:/root/go/bin:/root/.rbenv/bin:/root/.rbenv/shims:/root/.bun/bin:/root/.local/bin:/root/.pyenv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+++ export PATH
+++ '[' xscubaduck '!=' x ']'
+++ VIRTUAL_ENV_PROMPT=scubaduck
+++ export VIRTUAL_ENV_PROMPT
+++ '[' -z '' ']'
+++ '[' -z '' ']'
+++ _OLD_VIRTUAL_PS1=
+++ PS1='(scubaduck) '
+++ export PS1
+++ alias pydoc
+++ true
+++ hash -r
++ python -c 'import os; import duckdb; con = duckdb.connect(); con.execute(f"SET http_proxy = '\''{os.getenv("HTTP_PROXY")}'\''"); con.execute("INSTALL '\''sqlite'\'';")'
++ playwright install chromium
Downloading Chromium 136.0.7103.25 (playwright build v1169) from https://cdn.playwright.dev/dbazure/download/playwright/builds/chromium/1169/chromium-linux.zip
167.7 MiB [] 0% 0.0s167.7 MiB [] 0% 34.8s167.7 MiB [] 0% 24.3s167.7 MiB [] 0% 16.1s167.7 MiB [] 0% 10.2s167.7 MiB [] 1% 7.4s167.7 MiB [] 1% 6.6s167.7 MiB [] 1% 7.0s167.7 MiB [] 2% 6.4s167.7 MiB [] 3% 4.8s167.7 MiB [] 4% 3.6s167.7 MiB [] 5% 3.3s167.7 MiB [] 6% 3.2s167.7 MiB [] 7% 2.8s167.7 MiB [] 9% 2.4s167.7 MiB [] 10% 2.3s167.7 MiB [] 12% 2.0s167.7 MiB [] 14% 1.8s167.7 MiB [] 15% 1.8s167.7 MiB [] 17% 1.6s167.7 MiB [] 18% 1.6s167.7 MiB [] 19% 1.6s167.7 MiB [] 21% 1.4s167.7 MiB [] 23% 1.4s167.7 MiB [] 25% 1.3s167.7 MiB [] 26% 1.2s167.7 MiB [] 27% 1.2s167.7 MiB [] 29% 1.1s167.7 MiB [] 32% 1.0s167.7 MiB [] 33% 1.0s167.7 MiB [] 36% 0.9s167.7 MiB [] 38% 0.9s167.7 MiB [] 40% 0.8s167.7 MiB [] 41% 0.8s167.7 MiB [] 43% 0.8s167.7 MiB [] 44% 0.8s167.7 MiB [] 46% 0.7s167.7 MiB [] 47% 0.7s167.7 MiB [] 49% 0.7s167.7 MiB [] 51% 0.7s167.7 MiB [] 52% 0.6s167.7 MiB [] 54% 0.6s167.7 MiB [] 56% 0.6s167.7 MiB [] 59% 0.5s167.7 MiB [] 61% 0.5s167.7 MiB [] 62% 0.5s167.7 MiB [] 64% 0.4s167.7 MiB [] 66% 0.4s167.7 MiB [] 68% 0.4s167.7 MiB [] 71% 0.3s167.7 MiB [] 73% 0.3s167.7 MiB [] 76% 0.3s167.7 MiB [] 77% 0.3s167.7 MiB [] 78% 0.2s167.7 MiB [] 79% 0.2s167.7 MiB [] 81% 0.2s167.7 MiB [] 82% 0.2s167.7 MiB [] 83% 0.2s167.7 MiB [] 84% 0.2s167.7 MiB [] 87% 0.2s167.7 MiB [] 88% 0.1s167.7 MiB [] 90% 0.1s167.7 MiB [] 92% 0.1s167.7 MiB [] 93% 0.1s167.7 MiB [] 95% 0.1s167.7 MiB [] 96% 0.0s167.7 MiB [] 98% 0.0s167.7 MiB [] 100% 0.0s
Chromium 136.0.7103.25 (playwright build v1169) downloaded to /root/.cache/ms-playwright/chromium-1169
Downloading FFMPEG playwright build v1011 from https://cdn.playwright.dev/dbazure/download/playwright/builds/ffmpeg/1011/ffmpeg-linux.zip
2.3 MiB [] 0% 0.0s2.3 MiB [] 4% 0.4s2.3 MiB [] 12% 0.3s2.3 MiB [] 24% 0.2s2.3 MiB [] 48% 0.1s2.3 MiB [] 100% 0.0s
FFMPEG playwright build v1011 downloaded to /root/.cache/ms-playwright/ffmpeg-1011
Downloading Chromium Headless Shell 136.0.7103.25 (playwright build v1169) from https://cdn.playwright.dev/dbazure/download/playwright/builds/chromium/1169/chromium-headless-shell-linux.zip
101.4 MiB [] 0% 0.0s101.4 MiB [] 0% 20.0s101.4 MiB [] 0% 32.3s101.4 MiB [] 0% 16.4s101.4 MiB [] 1% 5.6s101.4 MiB [] 2% 3.6s101.4 MiB [] 4% 2.3s101.4 MiB [] 5% 2.0s101.4 MiB [] 7% 1.7s101.4 MiB [] 9% 1.5s101.4 MiB [] 10% 1.5s101.4 MiB [] 12% 1.3s101.4 MiB [] 15% 1.2s101.4 MiB [] 17% 1.1s101.4 MiB [] 21% 0.9s101.4 MiB [] 23% 0.9s101.4 MiB [] 26% 0.8s101.4 MiB [] 29% 0.7s101.4 MiB [] 33% 0.6s101.4 MiB [] 36% 0.6s101.4 MiB [] 38% 0.6s101.4 MiB [] 40% 0.5s101.4 MiB [] 41% 0.5s101.4 MiB [] 43% 0.5s101.4 MiB [] 44% 0.5s101.4 MiB [] 45% 0.5s101.4 MiB [] 47% 0.5s101.4 MiB [] 49% 0.5s101.4 MiB [] 50% 0.5s101.4 MiB [] 51% 0.5s101.4 MiB [] 52% 0.5s101.4 MiB [] 53% 0.5s101.4 MiB [] 54% 0.5s101.4 MiB [] 57% 0.5s101.4 MiB [] 59% 0.4s101.4 MiB [] 61% 0.4s101.4 MiB [] 64% 0.4s101.4 MiB [] 65% 0.4s101.4 MiB [] 68% 0.3s101.4 MiB [] 70% 0.3s101.4 MiB [] 71% 0.3s101.4 MiB [] 71% 0.4s101.4 MiB [] 72% 0.4s101.4 MiB [] 73% 0.3s101.4 MiB [] 77% 0.3s101.4 MiB [] 80% 0.2s101.4 MiB [] 83% 0.2s101.4 MiB [] 85% 0.2s101.4 MiB [] 86% 0.2s101.4 MiB [] 88% 0.1s101.4 MiB [] 89% 0.1s101.4 MiB [] 91% 0.1s101.4 MiB [] 96% 0.0s101.4 MiB [] 97% 0.0s101.4 MiB [] 98% 0.0s101.4 MiB [] 100% 0.0s
Chromium Headless Shell 136.0.7103.25 (playwright build v1169) downloaded to /root/.cache/ms-playwright/chromium_headless_shell-1169
++ echo 'source .venv/bin/activate'At the root of the repository, I'm checking the AGENTS.md file.
bash: .venv/bin/activate: No such file or directory
root@95eefccee427:/workspace# ls
scubaduckroot@95eefccee427:/workspace# cd scubaduck
root@95eefccee427:/workspace/scubaduck# ls
AGENTS.md codex_setup.sh scubaduck tests
README.md pyproject.toml stubs uv.lockroot@95eefccee427:/workspace/scubaduck# cat AGENTS.md
- Use "source .venv/bin/activate" to activate the virtual environment before
running any commands
- ONLY use non-interactive commands like cat, sed, apply_patch to do edits.
Do NOT use interactive editors.
- Do NOT attempt to install packages. Only the packages specified in
pyproject.toml are available. You cannot add new packages. If you
desperately want another package, make a note of it in the final PR
description.
- Use conventional commits to format PR title
- There are no nested AGENTS.md files, this is the only agents file
- When using Playwright, ONLY use chromium browser. This browser is already
installed.
- Use "ruff check" to check lint, "ruff format" to autoformat files and
"pyright" to typecheck.
- When writing the PR description, include the original prompt supplied by the
user.
- When you add functionality to the server, add server tests to
tests/test_server.py. When you add features to the frontend, add frontend
tests to tests/test_web.pyAGENTS says to follow certain steps, including running ruff check, pyright, and sticking to conventional commits.
root@95eefccee427:/workspace/scubaduck# ls
AGENTS.md codex_setup.sh scubaduck tests
README.md pyproject.toml stubs uv.lockroot@95eefccee427:/workspace/scubaduck# ls scubaduck
__init__.py sample.csv server.py staticroot@95eefccee427:/workspace/scubaduck# ls scubaduck/static
index.html jsroot@95eefccee427:/workspace/scubaduck# ls scubaduck/static/js
chip_input.js table_view.js timeseries_chart.js
sidebar.js time_column.js view_settings.jsI'll review the tests to confirm the status or any errors.
root@95eefccee427:/workspace/scubaduck# ls tests
conftest.py test_server_time.py test_web_graph.py
test_multi_table_web.py test_server_timeseries.py test_web_table.py
test_server.py test_web.py test_web_timeseries.py
test_server_basic.py test_web_basics.py web_utils.py
test_server_db_types.py test_web_chips.py
test_server_errors.py test_web_columns.pyroot@95eefccee427:/workspace/scubaduck# git status --shortI'll check the tests to understand their style and frameworks.
root@95eefccee427:/workspace/scubaduck# head -n 20 tests/test_server_timeseries.
py
from __future__ import annotations
import json
from typing import Any, cast
from scubaduck import server
def test_group_by_table() -> None:
app = server.app
client = app.test_client()
payload: dict[str, Any] = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "table",
"order_by": "user",
"limit": 10,
"columns": ["value"],
"group_by": ["user"],I'm moving down to capture the next part of the content.
root@95eefccee427:/workspace/scubaduck# sed -n '20,160p' tests/test_server_times
eries.py
"group_by": ["user"],
"aggregate": "Sum",
"show_hits": True,
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
rows = rv.get_json()["rows"]
assert rows[0][0] == "alice"
assert rows[0][1] == 2
assert rows[0][2] == 40
def test_table_avg_with_timestamp() -> None:
app = server.app
client = app.test_client()
payload: dict[str, Any] = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "table",
"order_by": "user",
"limit": 100,
"columns": ["user", "timestamp", "value"],
"group_by": ["user"],
"aggregate": "Avg",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert "error" not in data
rows = data["rows"]
assert rows[0][0] == "alice"
from dateutil import parser
ts = parser.parse(rows[0][2]).replace(tzinfo=None)
assert ts == parser.parse("2024-01-01 12:00:00")
def test_timeseries_basic() -> None:
app = server.app
client = app.test_client()
payload: dict[str, Any] = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "timeseries",
"limit": 100,
"group_by": ["user"],
"aggregate": "Count",
"columns": ["value"],
"x_axis": "timestamp",
"granularity": "1 day",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert len(data["rows"]) == 4
def test_timeseries_orders_by_xaxis() -> None:
app = server.app
client = app.test_client()
payload: dict[str, Any] = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "timeseries",
"limit": 100,
"columns": ["value"],
"x_axis": "timestamp",
"granularity": "1 day",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
rows = data["rows"]
from dateutil import parser
timestamps = [parser.parse(r[0]).replace(tzinfo=None) for r in rows]
assert timestamps == sorted(timestamps)
def test_timeseries_count_no_columns() -> None:
app = server.app
client = app.test_client()
payload: dict[str, Any] = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "timeseries",
"granularity": "1 day",
"columns": [],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
rows = data["rows"]
assert len(rows) == 2
assert rows[0][1] == 2
assert rows[1][1] == 2
assert len(rows[0]) == 3
def test_timeseries_limit_applies_to_series() -> None:
app = server.app
client = app.test_client()
payload: dict[str, Any] = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "timeseries",
"limit": 1,
"order_by": "user",
"group_by": ["user"],
"aggregate": "Count",
"columns": ["value"],
"x_axis": "timestamp",
"granularity": "1 day",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert len(data["rows"]) == 2
assert all(r[1] == "alice" for r in data["rows"])
def test_timeseries_auto_and_fine_buckets() -> None:
app = server.app
client = app.test_client()root@95eefccee427:/workspace/scubaduck# sed -n '160,320p' tests/test_server_time
series.py
def run(gran: str) -> None:
payload = {
"start": "2024-01-01 00:00:00",
"end": "2024-01-02 03:00:00",
"graph_type": "timeseries",
"columns": ["value"],
"x_axis": "timestamp",
"granularity": gran,
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/js
on"
)
data = rv.get_json()
assert rv.status_code == 200
from dateutil import parser
start = parser.parse(cast(str, payload["start"])).replace(tzinfo=None)
buckets = [
parser.parse(cast(str, r[0])).replace(tzinfo=None) for r in data["ro
ws"]
]
assert buckets[0] == start
if len(buckets) > 1:
step = (buckets[1] - buckets[0]).total_seconds()
assert step % data["bucket_size"] == 0
assert any(r[1] != 0 for r in data["rows"])
run("Auto")
run("Fine")
def test_timeseries_string_column_error() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "timeseries",
"limit": 7,
"columns": ["timestamp", "event", "value", "user"],
"x_axis": "timestamp",
"granularity": "1 hour",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
assert rv.status_code == 200
def test_derived_column_basic() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"limit": 10,
"columns": ["timestamp"],
"derived_columns": {"val2": "value * 2"},
"filters": [],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert data["rows"][0][1] == 20
def test_timeseries_derived_column() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "timeseries",
"granularity": "1 hour",
"limit": 7,
"columns": ["value"],
"derived_columns": {"derived_1": "value * 2"},
"aggregate": "Avg",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
rows = data["rows"]
assert all(r[3] == r[2] * 2 for r in rows)
def test_reserved_word_column() -> None:
app = server.create_app("TEST")
client = app.test_client()
payload = {
"table": "extra",
"columns": ["ts", "desc"],
"order_by": "ts",
"time_column": "",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert len(data["rows"]) == 2
assert data["rows"][0][1] == "x"
def test_order_by_samples_table() -> None:
app = server.app
client = app.test_client()
payload: dict[str, Any] = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "table",
"order_by": "Samples",
"order_dir": "DESC",
"limit": 10,
"columns": [],
"group_by": ["user"],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert 'ORDER BY "Hits" DESC' in data["sql"]
def test_order_by_samples_timeseries() -> None:
app = server.app
client = app.test_client()
payload: dict[str, Any] = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "timeseries",
"order_by": "Samples",
"order_dir": "DESC",
"x_axis": "timestamp",
"granularity": "1 day",
"columns": [],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert 'ORDER BY "Hits" DESC' in data["sql"]
def test_show_hits_client_side() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",I'll start by understanding how timeseries queries are processed in the server implementation.
root@95eefccee427:/workspace/scubaduck# sed -n '1,160p' scubaduck/server.py
from __future__ import annotations
from dataclasses import dataclass, field, replace
from typing import Any, Dict, List, Tuple, cast
import re
from datetime import datetime, timedelta, timezone
import time
from pathlib import Path
import os
import traceback
import math
import duckdb
from dateutil import parser as dtparser
from dateutil.relativedelta import relativedelta
from flask import Flask, jsonify, request, send_from_directory
def _quote(ident: str) -> str:
"""Return identifier quoted for SQL."""
return f'"{ident.replace('"', '""')}"'
@dataclass
class Filter:
column: str
op: str
value: str | int | float | list[str] | None
@dataclass
class QueryParams:
start: str | None = None
end: str | None = None
order_by: str | None = None
order_dir: str = "ASC"
limit: int | None = None
columns: list[str] = field(default_factory=lambda: [])
filters: list[Filter] = field(default_factory=lambda: [])
derived_columns: dict[str, str] = field(default_factory=lambda: {})
graph_type: str = "samples"
group_by: list[str] = field(default_factory=lambda: [])
aggregate: str | None = None
show_hits: bool = False
x_axis: str | None = None
granularity: str = "Auto"
fill: str = "0"
table: str = "events"
time_column: str | None = "timestamp"
time_unit: str = "s"
def _load_database(path: Path) -> duckdb.DuckDBPyConnection:
if not path.exists():
raise FileNotFoundError(path)
ext = path.suffix.lower()
if ext == ".csv":
con = duckdb.connect()
con.execute(
f"CREATE TABLE events AS SELECT * FROM read_csv_auto('{path.as_posix
()}')"
)
elif ext in {".db", ".sqlite"}:
con = duckdb.connect()
con.execute("LOAD sqlite")
con.execute(f"ATTACH '{path.as_posix()}' AS db (TYPE SQLITE)")
tables = [
r[0]
for r in con.execute(
"SELECT name FROM sqlite_master WHERE type='table'"
).fetchall()
]
for t in tables:
con.execute(f'CREATE VIEW "{t}" AS SELECT * FROM db."{t}"')
else:
con = duckdb.connect(path)
return con
def _create_test_database() -> duckdb.DuckDBPyConnection:
"""Return a DuckDB connection with a small multi-table dataset."""
con = duckdb.connect()
con.execute(
"CREATE TABLE events (id INTEGER PRIMARY KEY, ts INTEGER, val REAL, name
TEXT, flag BOOLEAN)"
)
con.execute("INSERT INTO events VALUES (1, 1704067200, 1.5, 'alice', 1)")
con.execute("INSERT INTO events VALUES (2, 1704070800, 2.0, 'bob', 0)")
con.execute('CREATE TABLE extra (ts INTEGER, "desc" TEXT, num INTEGER)')
con.execute("INSERT INTO extra VALUES (1704067200, 'x', 1)")
con.execute("INSERT INTO extra VALUES (1704070800, 'y', 2)")
return con
_REL_RE = re.compile(
r"([+-]?\d+(?:\.\d*)?)\s*(hour|hours|day|days|week|weeks|fortnight|fortnight
s|month|months|year|years)",
re.IGNORECASE,
)
def parse_time(val: str | None) -> str | None:
"""Parse an absolute or relative time string into ``YYYY-MM-DD HH:MM:SS``.""
"
if val is None or val == "":
return None
s = val.strip()
if s.lower() == "now":
dt = datetime.now(timezone.utc)
return dt.replace(microsecond=0).strftime("%Y-%m-%d %H:%M:%S")
m = _REL_RE.fullmatch(s)
if m:
qty = float(m.group(1))
unit = m.group(2).lower()
now = datetime.now(timezone.utc)
dt: datetime
if unit.startswith("hour"):
dt = now + timedelta(hours=qty)
elif unit.startswith("day"):
dt = now + timedelta(days=qty)
elif unit.startswith("week"):
dt = now + timedelta(weeks=qty)
elif unit.startswith("fortnight"):
dt = now + timedelta(weeks=2 * qty)
elif unit.startswith("month"):
if qty.is_integer():
dt = now + relativedelta(months=int(qty))
else:
dt = now + timedelta(days=30 * qty)
elif unit.startswith("year"):
if qty.is_integer():
dt = now + relativedelta(years=int(qty))
else:
dt = now + timedelta(days=365 * qty)
else: # pragma: no cover - defensive
raise ValueError(f"Unsupported unit: {unit}")
return dt.replace(microsecond=0).strftime("%Y-%m-%d %H:%M:%S")
dt = dtparser.parse(s)
return dt.replace(microsecond=0, tzinfo=None).strftime("%Y-%m-%d %H:%M:%S")
def _numeric_to_datetime(value: int | float, unit: str) -> datetime:
"""Convert a numeric timestamp ``value`` with unit ``unit`` to ``datetime``.
Heuristically fall back to seconds when the converted value is before 1990
but the seconds interpretation is in a reasonable range. This handles
integer columns stored in seconds even when ``unit`` is mistakenly set to a
finer granularity.
"""
divisor = {
"s": 1,
"ms": 1000,
"us": 1_000_000,
"ns": 1_000_000_000,
}.get(unit, 1)
dt = datetime.fromtimestamp(int(value) / divisor, tz=timezone.utc)
if unit != "s" and dt.year < 1990:root@95eefccee427:/workspace/scubaduck# sed -n '160,320p' scubaduck/server.py
if unit != "s" and dt.year < 1990:
alt = datetime.fromtimestamp(int(value), tz=timezone.utc)
if alt.year >= 1990:
dt = alt
return dt
def _granularity_seconds(granularity: str, start: str | None, end: str | None) -
> int:
gran = granularity.lower()
mapping = {
"1 second": 1,
"5 seconds": 5,
"10 seconds": 10,
"30 seconds": 30,
"1 minute": 60,
"4 minutes": 240,
"5 minutes": 300,
"10 minutes": 600,
"15 minutes": 900,
"30 minutes": 1800,
"1 hour": 3600,
"3 hours": 10800,
"6 hours": 21600,
"1 day": 86400,
"1 week": 604800,
"30 days": 2592000,
}
if gran in mapping:
return mapping[gran]
if gran in {"auto", "fine"} and start and end:
try:
s = dtparser.parse(start)
e = dtparser.parse(end)
except Exception:
return 3600
total = max((e - s).total_seconds(), 1)
buckets = 100 if gran == "auto" else 500
return max(int(total // buckets), 1)
return 3600
def _time_expr(col: str, column_types: Dict[str, str] | None, unit: str) -> str:
"""Return SQL expression for column interpreted as timestamp."""
qcol = _quote(col)
if column_types is None:
return qcol
ctype = column_types.get(col, "").upper()
if not any(t in ctype for t in ["TIMESTAMP", "DATE", "TIME"]):
if any(
t in ctype
for t in [
"INT",
"DECIMAL",
"REAL",
"DOUBLE",
"FLOAT",
"NUMERIC",
"HUGEINT",
]
):
if unit == "ns":
# Use nanosecond helper unless column cannot represent such larg
e values
if "INT" in ctype and "BIGINT" not in ctype and "HUGEINT" not in
ctype:
unit = "s"
else:
expr = f"CAST({qcol} AS BIGINT)"
return f"make_timestamp_ns({expr})"
if (
unit != "s"
and "INT" in ctype
and "BIGINT" not in ctype
and "HUGEINT" not in ctype
):
# 32-bit integers cannot store sub-second precision for modern d
ates
unit = "s"
multiplier = {
"s": 1_000_000,
"ms": 1_000,
"us": 1,
}.get(unit, 1_000_000)
base = f"CAST({qcol} AS BIGINT)"
expr = f"CAST({base} * {multiplier} AS BIGINT)" if multiplier != 1 e
lse base
return f"make_timestamp({expr})"
return qcol
def build_query(params: QueryParams, column_types: Dict[str, str] | None = None)
-> str:
select_parts: list[str] = []
group_cols = params.group_by[:]
selected_for_order = set(params.columns) | set(params.derived_columns.keys()
)
if params.graph_type == "timeseries":
sec = _granularity_seconds(params.granularity, params.start, params.end)
x_axis = params.x_axis or params.time_column
if x_axis is None:
raise ValueError("x_axis required for timeseries")
xexpr = _time_expr(x_axis, column_types, params.time_unit)
if params.start:
bucket_expr = (
f"TIMESTAMP '{params.start}' + INTERVAL '{sec} second' * "
f"CAST(floor((epoch({xexpr}) - epoch(TIMESTAMP '{params.start}')
)/{sec}) AS BIGINT)"
)
else:
bucket_expr = (
f"TIMESTAMP 'epoch' + INTERVAL '{sec} second' * "
f"CAST(floor(epoch({xexpr})/{sec}) AS BIGINT)"
)
select_parts.append(f"{bucket_expr} AS bucket")
group_cols = ["bucket"] + group_cols
selected_for_order.add("bucket")
has_agg = bool(group_cols) or params.aggregate is not None
if has_agg:
select_cols = (
group_cols[1:] if params.graph_type == "timeseries" else group_cols
)
select_parts.extend(_quote(c) for c in select_cols)
agg = (params.aggregate or "count").lower()
selected_for_order.update(group_cols)
def agg_expr(col: str) -> str:
expr = _quote(col)
ctype = column_types.get(col, "").upper() if column_types else ""
if "BOOL" in ctype:
expr = f"CAST({_quote(col)} AS BIGINT)"
if agg.startswith("p"):
quant = float(agg[1:]) / 100
return f"quantile({expr}, {quant})"
if agg == "count distinct":
return f"count(DISTINCT {expr})"
if agg == "avg" and column_types is not None:
if "TIMESTAMP" in ctype or "DATE" in ctype or "TIME" in ctype:
return (
"TIMESTAMP 'epoch' + INTERVAL '1 second' * "
f"CAST(avg(epoch({_quote(col)})) AS BIGINT)"
)
return f"{agg}({expr})"
if agg == "count":
select_parts.append("count(*) AS Count")
selected_for_order.add("Count")
else:
for col in params.columns:
if col in group_cols:
continue
select_parts.append(f"{agg_expr(col)} AS {_quote(col)}")
selected_for_order.add(col)
select_parts.insert(len(group_cols), "count(*) AS Hits")
selected_for_order.add("Hits")
else:
select_parts.extend(_quote(c) for c in params.columns)
selected_for_order.update(params.columns)
order_by = params.order_by
if order_by == "Samples":
order_by = "Hits"
order_by = order_by if order_by in selected_for_order else None
if has_agg and params.derived_columns:
inner_params = replace(
params,root@95eefccee427:/workspace/scubaduck# sed -n '320,640p' scubaduck/server.py
params,
derived_columns={},
order_by=None,
limit=None,
)
inner_sql = build_query(inner_params, column_types)
outer_select = ["t.*"] + [
f"{expr} AS {name}" for name, expr in params.derived_columns.items()
]
indented_inner = "\n".join(" " + line for line in inner_sql.splitline
s())
lines = [
f"SELECT {', '.join(outer_select)}",
"FROM (",
indented_inner,
") t",
]
if order_by:
lines.append(f"ORDER BY {_quote(order_by)} {params.order_dir}")
elif params.graph_type == "timeseries":
lines.append("ORDER BY bucket")
if params.limit is not None:
lines.append(f"LIMIT {params.limit}")
return "\n".join(lines)
for name, expr in params.derived_columns.items():
select_parts.append(f"{expr} AS {name}")
selected_for_order.add(name)
select_clause = ", ".join(select_parts) if select_parts else "*"
lines = [f"SELECT {select_clause}", f'FROM "{params.table}"']
where_parts: list[str] = []
if params.time_column:
time_expr = _time_expr(params.time_column, column_types, params.time_uni
t)
else:
time_expr = None
if time_expr and params.start:
where_parts.append(f"{time_expr} >= '{params.start}'")
if time_expr and params.end:
where_parts.append(f"{time_expr} <= '{params.end}'")
for f in params.filters:
op = f.op
if op in {"empty", "!empty"}:
val = "''"
else:
if f.value is None:
continue
if isinstance(f.value, list):
if not f.value:
continue
if op == "=":
qcol = _quote(f.column)
vals = " OR ".join(f"{qcol} = '{v}'" for v in f.value)
where_parts.append(f"({vals})")
continue
val = f"'{f.value}'" if isinstance(f.value, str) else str(f.value)
qcol = _quote(f.column)
if op == "contains":
where_parts.append(f"{qcol} ILIKE '%' || {val} || '%'")
elif op == "!contains":
where_parts.append(f"{qcol} NOT ILIKE '%' || {val} || '%'")
elif op == "empty":
where_parts.append(f"{qcol} = {val}")
elif op == "!empty":
where_parts.append(f"{qcol} != {val}")
else:
where_parts.append(f"{qcol} {op} {val}")
if where_parts:
lines.append("WHERE " + " AND ".join(where_parts))
if group_cols:
lines.append("GROUP BY " + ", ".join(_quote(c) for c in group_cols))
if order_by:
lines.append(f"ORDER BY {_quote(order_by)} {params.order_dir}")
elif params.graph_type == "timeseries":
lines.append("ORDER BY bucket")
if params.limit is not None:
lines.append(f"LIMIT {params.limit}")
return "\n".join(lines)
def create_app(db_file: str | Path | None = None) -> Flask:
app = Flask(__name__, static_folder="static")
if db_file is None:
env_db = os.environ.get("SCUBADUCK_DB")
if env_db:
db_file = env_db
if isinstance(db_file, str) and db_file.upper() == "TEST":
con = _create_test_database()
else:
db_path = Path(db_file or Path(__file__).with_name("sample.csv")).resolv
e()
con = _load_database(db_path)
tables = [r[0] for r in con.execute("SHOW TABLES").fetchall()]
if not tables:
raise ValueError("No tables found in database")
default_table = tables[0]
columns_cache: Dict[str, Dict[str, str]] = {}
def get_columns(table: str) -> Dict[str, str]:
if table not in columns_cache:
rows = con.execute(f'PRAGMA table_info("{table}")').fetchall()
if not rows:
raise ValueError(f"Unknown table: {table}")
columns_cache[table] = {r[1]: r[2] for r in rows}
return columns_cache[table]
sample_cache: Dict[Tuple[str, str, str], Tuple[List[str], float]] = {}
CACHE_TTL = 60.0
CACHE_LIMIT = 200
@app.route("/")
def index() -> Any: # pyright: ignore[reportUnusedFunction]
assert app.static_folder is not None
return send_from_directory(app.static_folder, "index.html")
@app.route("/js/<path:filename>")
def js(filename: str) -> Any: # pyright: ignore[reportUnusedFunction]
assert app.static_folder is not None
folder = Path(app.static_folder) / "js"
return send_from_directory(folder, filename)
@app.route("/api/tables")
...
if params.table not in tables:
return jsonify({"error": "Invalid table"}), 400
column_types = get_columns(params.table)
if params.time_column and params.time_column not in column_types:
return jsonify({"error": "Invalid time_column"}), 400
if params.time_unit not in {"s", "ms", "us", "ns"}:
return jsonify({"error": "Invalid time_unit"}), 400
if params.graph_type not in {"table", "timeseries"} and (
params.group_by or params.aggregate or params.show_hits
):
return (
jsonify(
{
"error": "group_by, aggregate and show_hits are only val
id for table or timeseries view"
}
),
400,
)
valid_cols = set(column_types.keys())
valid_cols.update(params.derived_columns.keys())
valid_cols.add("Hits")
if params.graph_type == "timeseries":
if params.x_axis is None:
params.x_axis = params.time_column
if params.x_axis is None or params.x_axis not in valid_cols:
return jsonify({"error": "Invalid x_axis"}), 400
ctype = column_types.get(params.x_axis, "").upper()
is_time = any(t in ctype for t in ["TIMESTAMP", "DATE", "TIME"])
is_numeric = any(
t in ctype
for t in [
"INT",
"DECIMAL",
"REAL",
"DOUBLE",
"FLOAT",
"NUMERIC",
"HUGEINT",
]
)
if not (is_time or is_numeric):
return jsonify({"error": "x_axis must be a time column"}), 400
for col in params.columns:
if col not in valid_cols:
return jsonify({"error": f"Unknown column: {col}"}), 400
for col in params.group_by:
if col not in valid_cols:
return jsonify({"error": f"Unknown column: {col}"}), 400
if params.order_by and params.order_by not in valid_cols:
return jsonify({"error": f"Unknown column: {params.order_by}"}), 400
if params.group_by or params.graph_type == "timeseries":
agg = (params.aggregate or "count").lower()
if agg.startswith("p") or agg == "sum":
need_numeric = True
allow_time = False
elif agg == "avg" or agg in {"min", "max"}:
need_numeric = False
allow_time = True
else:
need_numeric = False
allow_time = False
if need_numeric or allow_time:
for c in params.columns:
if c in params.group_by or c == params.x_axis:
continue
if c not in column_types:
continue
ctype = column_types.get(c, "").upper()
is_numeric = "BOOL" in ctype or any(
t in ctype
for t in [
"INT",
"DECIMAL",
"REAL",
"DOUBLE",
"FLOAT",
"NUMERIC",
"HUGEINT",
]
)
is_time = "TIMESTAMP" in ctype or "DATE" in ctype or "TIME"
in ctype
if need_numeric and not is_numeric:
return (
jsonify(
{
"error": f"Aggregate {agg} cannot be applied
to column {c}",
}
),
400,
)
if allow_time and not (is_numeric or is_time):
return (
jsonify(
{
"error": f"Aggregate {agg} cannot be applied
to column {c}",
}
),
400,
)
if (params.start is None or params.end is None) and (
params.x_axis or params.time_column
):
axis = params.x_axis or params.time_column
assert axis is not None
row = cast(
tuple[datetime | None, datetime | None],
con.execute(
f'SELECT min({_quote(axis)}), max({_quote(axis)}) FROM "{par
ams.table}"'
).fetchall()[0],
)
mn, mx = row
if isinstance(mn, (int, float)):root@95eefccee427:/workspace/scubaduck# sed -n '640,960p' scubaduck/server.py
if isinstance(mn, (int, float)):
try:
mn = _numeric_to_datetime(mn, params.time_unit)
except Exception:
return (
jsonify(
{
"error": (
f"Invalid time value {mn} for column {axis}"
f" with time_unit {params.time_unit}"
)
}
),
400,
)
if isinstance(mx, (int, float)):
try:
mx = _numeric_to_datetime(mx, params.time_unit)
except Exception:
return (
jsonify(
{
"error": (
f"Invalid time value {mx} for column {axis}"
f" with time_unit {params.time_unit}"
)
}
),
400,
)
if params.start is None and mn is not None:
params.start = (
mn.strftime("%Y-%m-%d %H:%M:%S") if not isinstance(mn, str)
else mn
)
if params.end is None and mx is not None:
params.end = (
mx.strftime("%Y-%m-%d %H:%M:%S") if not isinstance(mx, str)
else mx
)
bucket_size: int | None = None
if params.graph_type == "timeseries":
bucket_size = _granularity_seconds(
params.granularity,
params.start if isinstance(params.start, str) else None,
params.end if isinstance(params.end, str) else None,
)
if (
params.limit is not None
and params.start is not None
and params.end is not None
):
try:
start_dt = dtparser.parse(params.start)
end_dt = dtparser.parse(params.end)
buckets = math.ceil(
(end_dt - start_dt).total_seconds() / bucket_size
)
if buckets > 1:
params.limit *= buckets
except Exception:
pass
sql = build_query(params, column_types)
try:
rows = con.execute(sql).fetchall()
except Exception as exc:
tb = traceback.format_exc()
print(f"Query failed:\n{sql}\n{tb}")
return (
jsonify({"sql": sql, "error": str(exc), "traceback": tb}),
400,
)
result: Dict[str, Any] = {"sql": sql, "rows": rows}
if params.start is not None:
result["start"] = str(params.start)
if params.end is not None:
result["end"] = str(params.end)
if bucket_size is not None:
result["bucket_size"] = bucket_size
return jsonify(result)
return app
app = create_app()
if __name__ == "__main__":
app.run(debug=True)Looks like there's an error. I'll search tests using "time_unit" or "ts".
root@95eefccee427:/workspace/scubaduck# grep -n "time_unit" -r tests | head
tests/test_server_time.py:33:def test_integer_time_unit_ms(tmp_path: Path) -> No
ne:
tests/test_server_time.py:45: "time_unit": "ms",
tests/test_server_time.py:69: "time_unit": "ms",
tests/test_server_time.py:79:def test_integer_time_unit_us_default_start_end(tmp
_path: Path) -> None:
tests/test_server_time.py:91: "time_unit": "us",
tests/test_server_time.py:103:def test_sqlite_integer_time_unit_us(tmp_path: Pat
h) -> None:
tests/test_server_time.py:123: "time_unit": "us",
tests/test_server_time.py:239: "time_unit": "s",
tests/test_server_time.py:257: "time_unit": "us",
tests/test_web_basics.py:77:def test_time_unit_dropdown(page: Any, server_url: s
tr) -> None:I'll check tests/test_server_time.py for a similar scenario to guide my approach.
from scubaduck import server
def test_integer_time_column(tmp_path: Path) -> None:
csv_file = tmp_path / "events.csv"
csv_file.write_text("created,event\n1704067200,login\n1704070800,logout\n")
app = server.create_app(csv_file)
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-01 01:00:00",
"order_by": "created",
"columns": ["created", "event"],
"time_column": "created",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert len(data["rows"]) == 2
def test_integer_time_unit_ms(tmp_path: Path) -> None:
csv_file = tmp_path / "events.csv"
csv_file.write_text("created,event\n1704067200000,login\n1704070800000,logou
t\n")
app = server.create_app(csv_file)
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-01 01:00:00",
"order_by": "created",
"columns": ["created", "event"],
"time_column": "created",
"time_unit": "ms",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert len(data["rows"]) == 2
def test_timeseries_default_xaxis_uses_time_column(tmp_path: Path) -> None:
csv_file = tmp_path / "events.csv"
csv_file.write_text("created,event\n1704067200000,login\n1704070800000,logou
t\n")
app = server.create_app(csv_file)
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-01 01:00:00",
"graph_type": "timeseries",
"granularity": "1 hour",
"columns": ["event"],
"aggregate": "Count",
"time_column": "created",
"time_unit": "ms",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert len(data["rows"]) == 2
def test_integer_time_unit_us_default_start_end(tmp_path: Path) -> None:
csv_file = tmp_path / "events.csv"
csv_file.write_text(
"created,event\n1704067200000000,login\n1704070800000000,logout\n"
)
app = server.create_app(csv_file)
client = app.test_client()
payload = {
"table": "events",
"order_by": "created",
"columns": ["created", "event"],
"time_column": "created",
"time_unit": "us",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert data["start"] == "2024-01-01 00:00:00"
assert data["end"] == "2024-01-01 01:00:00"
assert len(data["rows"]) == 2
def test_sqlite_integer_time_unit_us(tmp_path: Path) -> None:
sqlite_file = tmp_path / "events.sqlite"
import sqlite3
conn = sqlite3.connect(sqlite_file)
conn.execute("CREATE TABLE visits (visit_time INTEGER, event TEXT)")
big_ts = 13384551652000000
conn.execute("INSERT INTO visits VALUES (?, ?)", (big_ts, "foo"))
conn.commit()
conn.close() # pyright: ignore[reportUnknownMemberType, reportAttributeAcce
ssIssue]
app = server.create_app(sqlite_file)
client = app.test_client()
payload = {
"table": "visits",
"start": "2394-02-20 00:00:00",
"end": "2394-02-21 00:00:00",
"order_by": "visit_time",
"columns": ["visit_time", "event"],
"time_column": "visit_time",
"time_unit": "us",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert len(data["rows"]) == 1
def test_relative_time_query(monkeypatch: pytest.MonkeyPatch) -> None:
app = server.app
client = app.test_client()
from datetime import datetime
fixed_now = datetime(2024, 1, 2, 4, 0, 0)
class FixedDateTime(datetime):
@classmethod
def now(cls, tz=None): # type: ignore[override]
return fixed_now if tz is None else fixed_now.astimezone(tz)
monkeypatch.setattr(server, "datetime", FixedDateTime)
payload = {
"table": "events",
"start": "-1 hour",
"end": "now",
"order_by": "timestamp",
"limit": 100,
"columns": ["timestamp", "event", "value", "user"],
"filters": [],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()I'm scrolling to locate the timeseries tests in the code.
root@95eefccee427:/workspace/scubaduck# grep -n "timeseries" -n tests/test_serve
r_time.py | head
55:def test_timeseries_default_xaxis_uses_time_column(tmp_path: Path) -> None:
64: "graph_type": "timeseries",I’ll check beyond line 200 to see if there’s anything more relevant.
assert len(data["rows"]) == 1
assert data["rows"][0][3] == "charlie"
def test_relative_month_year(monkeypatch: pytest.MonkeyPatch) -> None:
app = server.app
client = app.test_client()
from datetime import datetime
fixed_now = datetime(2024, 1, 2, 0, 0, 0)
class FixedDateTime(datetime):
@classmethod
def now(cls, tz=None): # type: ignore[override]
return fixed_now if tz is None else fixed_now.astimezone(tz)
monkeypatch.setattr(server, "datetime", FixedDateTime)
payload = {
"table": "events",
"start": "-1 year",
"end": "-1 month",
"order_by": "timestamp",
"limit": 10,
"columns": ["timestamp"],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert data["start"] == "2023-01-02 00:00:00"
assert data["end"] == "2023-12-02 00:00:00"
def test_default_start_end_returned() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"order_by": "timestamp",
"limit": 5,
"columns": ["timestamp"],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert data["start"] == "2024-01-01 00:00:00"
assert data["end"] == "2024-01-02 03:00:00"
def test_time_column_none_no_time_filter() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"columns": ["timestamp", "event"],
"time_column": "",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert "start" not in data and "end" not in data
assert len(data["rows"]) == 4
def test_test_dataset_int32_time_s() -> None:
app = server.create_app("TEST")
client = app.test_client()
payload = {
"table": "events",
"time_column": "ts",
"time_unit": "s",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert data["start"] == "2024-01-01 00:00:00"
assert data["end"] == "2024-01-01 01:00:00"
assert len(data["rows"]) == 2
def test_test_dataset_int32_time_us() -> None:
app = server.create_app("TEST")
client = app.test_client()
payload = {
"table": "events",
"time_column": "ts",
"time_unit": "us",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert data["start"] == "2024-01-01 00:00:00"
assert data["end"] == "2024-01-01 01:00:00"
assert len(data["rows"]) == 2I need to scroll further to view the test for the timeseries integer unit.
root@95eefccee427:/workspace/scubaduck# grep -n "timeseries" tests/test_server_t
ime.py
55:def test_timeseries_default_xaxis_uses_time_column(tmp_path: Path) -> None:
64: "graph_type": "timeseries",It looks like the bug occurs with blanks in time-series graphs when no columns selected.
duck/server.py
82:def _create_test_database() -> duckdb.DuckDBPyConnection:
406: con = _create_test_database()def _create_test_database() -> duckdb.DuckDBPyConnection:
"""Return a DuckDB connection with a small multi-table dataset."""
con = duckdb.connect()
con.execute(
"CREATE TABLE events (id INTEGER PRIMARY KEY, ts INTEGER, val REAL, name
TEXT, flag BOOLEAN)"
)
con.execute("INSERT INTO events VALUES (1, 1704067200, 1.5, 'alice', 1)")
con.execute("INSERT INTO events VALUES (2, 1704070800, 2.0, 'bob', 0)")
con.execute('CREATE TABLE extra (ts INTEGER, "desc" TEXT, num INTEGER)')
con.execute("INSERT INTO extra VALUES (1704067200, 'x', 1)")
con.execute("INSERT INTO extra VALUES (1704070800, 'y', 2)")
return con
_REL_RE = re.compile(
r"([+-]?\d+(?:\.\d*)?)\s*(hour|hours|day|days|week|weeks|fortnight|fortnight
s|month|months|year|years)",
re.IGNORECASE,
)
def parse_time(val: str | None) -> str | None:
"""Parse an absolute or relative time string into ``YYYY-MM-DD HH:MM:SS``.""
"
if val is None or val == "":
return None
s = val.strip()
if s.lower() == "now":
dt = datetime.now(timezone.utc)
return dt.replace(microsecond=0).strftime("%Y-%m-%d %H:%M:%S")I'm checking the test for missing time series scenario with empty columns, confirming SQL behavior.
"""Return SQL expression for column interpreted as timestamp."""
qcol = _quote(col)
if column_types is None:
return qcol
ctype = column_types.get(col, "").upper()
if not any(t in ctype for t in ["TIMESTAMP", "DATE", "TIME"]):
if any(
t in ctype
for t in [
"INT",
"DECIMAL",
"REAL",
"DOUBLE",
"FLOAT",
"NUMERIC",
"HUGEINT",
]
):
if unit == "ns":
# Use nanosecond helper unless column cannot represent such larg
e values
if "INT" in ctype and "BIGINT" not in ctype and "HUGEINT" not in
ctype:
unit = "s"
else:
expr = f"CAST({qcol} AS BIGINT)"
return f"make_timestamp_ns({expr})"
if (
unit != "s"
and "INT" in ctype
and "BIGINT" not in ctype
and "HUGEINT" not in ctype
):
# 32-bit integers cannot store sub-second precision for modern d
ates
unit = "s"
multiplier = {
"s": 1_000_000,
"ms": 1_000,
"us": 1,
}.get(unit, 1_000_000)
base = f"CAST({qcol} AS BIGINT)"
expr = f"CAST({base} * {multiplier} AS BIGINT)" if multiplier != 1 e
lse base
return f"make_timestamp({expr})"
return qcol
def build_query(params: QueryParams, column_types: Dict[str, str] | None = None)
-> str:
select_parts: list[str] = []
group_cols = params.group_by[:]
selected_for_order = set(params.columns) | set(params.derived_columns.keys()
)
if params.graph_type == "timeseries":
sec = _granularity_seconds(params.granularity, params.start, params.end)
x_axis = params.x_axis or params.time_column
if x_axis is None:
raise ValueError("x_axis required for timeseries")
xexpr = _time_expr(x_axis, column_types, params.time_unit)
if params.start:
bucket_expr = (
f"TIMESTAMP '{params.start}' + INTERVAL '{sec} second' * "
f"CAST(floor((epoch({xexpr}) - epoch(TIMESTAMP '{params.start}')
)/{sec}) AS BIGINT)"
)
else:
bucket_expr = (
f"TIMESTAMP 'epoch' + INTERVAL '{sec} second' * "
f"CAST(floor(epoch({xexpr})/{sec}) AS BIGINT)"
)
select_parts.append(f"{bucket_expr} AS bucket")
group_cols = ["bucket"] + group_cols
selected_for_order.add("bucket")
has_agg = bool(group_cols) or params.aggregate is not None
if has_agg:
select_cols = (
group_cols[1:] if params.graph_type == "timeseries" else group_cols
)
select_parts.extend(_quote(c) for c in select_cols)
agg = (params.aggregate or "count").lower()
selected_for_order.update(group_cols)
def agg_expr(col: str) -> str:root@95eefccee427:/workspace/scubaduck# sed -n '280,360p' scubaduck/server.py
def agg_expr(col: str) -> str:
expr = _quote(col)
ctype = column_types.get(col, "").upper() if column_types else ""
if "BOOL" in ctype:
expr = f"CAST({_quote(col)} AS BIGINT)"
if agg.startswith("p"):
quant = float(agg[1:]) / 100
return f"quantile({expr}, {quant})"
if agg == "count distinct":
return f"count(DISTINCT {expr})"
if agg == "avg" and column_types is not None:
if "TIMESTAMP" in ctype or "DATE" in ctype or "TIME" in ctype:
return (
"TIMESTAMP 'epoch' + INTERVAL '1 second' * "
f"CAST(avg(epoch({_quote(col)})) AS BIGINT)"
)
return f"{agg}({expr})"
if agg == "count":
select_parts.append("count(*) AS Count")
selected_for_order.add("Count")
else:
for col in params.columns:
if col in group_cols:
continue
select_parts.append(f"{agg_expr(col)} AS {_quote(col)}")
selected_for_order.add(col)
select_parts.insert(len(group_cols), "count(*) AS Hits")
selected_for_order.add("Hits")
else:
select_parts.extend(_quote(c) for c in params.columns)
selected_for_order.update(params.columns)
order_by = params.order_by
if order_by == "Samples":
order_by = "Hits"
order_by = order_by if order_by in selected_for_order else None
if has_agg and params.derived_columns:
inner_params = replace(
params,
derived_columns={},
order_by=None,
limit=None,
)
inner_sql = build_query(inner_params, column_types)
outer_select = ["t.*"] + [
f"{expr} AS {name}" for name, expr in params.derived_columns.items()
]
indented_inner = "\n".join(" " + line for line in inner_sql.splitline
s())
lines = [
f"SELECT {', '.join(outer_select)}",
"FROM (",
indented_inner,
") t",
]
if order_by:
lines.append(f"ORDER BY {_quote(order_by)} {params.order_dir}")
elif params.graph_type == "timeseries":
lines.append("ORDER BY bucket")
if params.limit is not None:
lines.append(f"LIMIT {params.limit}")
return "\n".join(lines)
for name, expr in params.derived_columns.items():
select_parts.append(f"{expr} AS {name}")
selected_for_order.add(name)
select_clause = ", ".join(select_parts) if select_parts else "*"
lines = [f"SELECT {select_clause}", f'FROM "{params.table}"']
where_parts: list[str] = []
if params.time_column:
time_expr = _time_expr(params.time_column, column_types, params.time_uni
t)
else:
time_expr = None
if time_expr and params.start:
where_parts.append(f"{time_expr} >= '{params.start}'")
if time_expr and params.end:
where_parts.append(f"{time_expr} <= '{params.end}'")
for f in params.filters:
op = f.op
if op in {"empty", "!empty"}:The user's bug report suggests an issue with SQL aggregation due to missing columns. I'll inspect the SQL.
root@95eefccee427:/workspace/scubaduck# source .venv/bin/activate(scubaduck) root@95eefccee427:/workspace/scubaduck# python - <<'PY'
> from scubaduck import server
> app = server.create_app('TEST')
> client = app.test_client()
> payload = {
> "table": "events",
> "graph_type": "timeseries",
> "time_column": "ts",
> "time_unit": "s",
> "aggregate": "Count",
> "columns": [],
> "granularity": "1 hour",
> }
> rv = client.post("/api/query", json=payload)
> print('status', rv.status_code)
> print(rv.get_json())
> PYstatus 200
{'bucket_size': 3600, 'end': '2024-01-01 01:00:00', 'rows': [['Mon, 01 Jan 2024
00:00:00 GMT', 1, 1], ['Mon, 01 Jan 2024 01:00:00 GMT', 1, 1]], 'sql': 'SELECT T
IMESTAMP \'2024-01-01 00:00:00\' + INTERVAL \'3600 second\' * CAST(floor((epoch(
make_timestamp(CAST(CAST("ts" AS BIGINT) * 1000000 AS BIGINT))) - epoch(TIMESTAM
P \'2024-01-01 00:00:00\'))/3600) AS BIGINT) AS bucket, count(*) AS Hits, count(
*) AS Count\nFROM "events"\nWHERE make_timestamp(CAST(CAST("ts" AS BIGINT) * 100
0000 AS BIGINT)) >= \'2024-01-01 00:00:00\' AND make_timestamp(CAST(CAST("ts" AS
BIGINT) * 1000000 AS BIGINT)) <= \'2024-01-01 01:00:00\'\nGROUP BY "bucket"\nOR
DER BY bucket', 'start': '2024-01-01 00:00:00'}
(scubaduck) root@95eefccee427:/workspace/scubaduck#It seems the frontend expects at least one column besides "Hits" for the chart.
(scubaduck) root@95eefccee427:/workspace/scubaduck# ls scubaduck/static/js
chip_input.js table_view.js timeseries_chart.js
sidebar.js time_column.js view_settings.js(scubaduck) root@95eefccee427:/workspace/scubaduck# sed -n '1,200p' scubaduck/st
atic/js/timeseries_chart.js
let resizeObserver = null;
let currentChart = null;
function showTimeSeries(data) {
function parseTs(s) {
if (s.match(/GMT/) || s.endsWith('Z') || /\+\d{2}:?\d{2}$/.test(s)) {
return new Date(s).getTime();
}
return new Date(s + 'Z').getTime();
}
const view = document.getElementById('view');
if (data.rows.length === 0) {
view.innerHTML = '<p id="empty-message">Empty data provided to table</p>';
return;
}
const height = 600;
view.innerHTML =
'<div id="ts-container"><div id="legend"></div><div id="chart-wrapper"><svg
id="chart" height="' +
height +
'"></svg></div></div>';
const svg = document.getElementById('chart');
const legend = document.getElementById('legend');
const crosshairLine = document.createElementNS('http://www.w3.org/2000/svg', '
line');
crosshairLine.id = 'crosshair_line';
crosshairLine.setAttribute('stroke', '#555');
crosshairLine.style.display = 'none';
const crosshairDots = document.createElementNS('http://www.w3.org/2000/svg', '
g');
crosshairDots.id = 'crosshair_dots';
crosshairDots.style.display = 'none';
const groups = groupBy.chips || [];
const hasHits = document.getElementById('show_hits').checked ? 1 : 0;
const fill = document.getElementById('fill').value;
const bucketMs = (data.bucket_size || 3600) * 1000;
const start = data.start ? parseTs(data.start) : null;
const end = data.end ? parseTs(data.end) : null;
const startIdx = 1 + groups.length + hasHits;
const valueCols = selectedColumns.slice(groups.length + hasHits);
const series = {};
data.rows.forEach(r => {
const ts = parseTs(r[0]);
const groupKey = groups.map((_, i) => r[1 + i]).join(':') || 'all';
valueCols.forEach((name, i) => {
const val = Number(r[startIdx + i]);
const key = groupKey === 'all' ? name : groupKey + ':' + name;
if (!series[key]) series[key] = {};
series[key][ts] = val;
});
});
const buckets = [];
let minX = start !== null ? start : Infinity;
let maxX = end !== null ? end : -Infinity;
if (start !== null && end !== null) {
for (let t = start; t <= end; t += bucketMs) {
buckets.push(t);
}
} else {
Object.keys(series).forEach(k => {
const s = series[k];
Object.keys(s).forEach(t => {
const n = Number(t);
if (n < minX) minX = n;
if (n > maxX) maxX = n;
});
});
for (let t = minX; t <= maxX; t += bucketMs) {
buckets.push(t);
}
}
let minY = Infinity,
maxY = -Infinity;
Object.keys(series).forEach(key => {
const vals = series[key];
buckets.forEach(b => {
const v = vals[b];
const val = v === undefined && fill === '0' ? 0 : v;
if (val === undefined) return;
if (val < minY) minY = val;
if (val > maxY) maxY = val;
});
});
if (fill === '0') {
if (minY > 0) minY = 0;
if (maxY < 0) maxY = 0;
}
const colors = [
'#1f77b4',
'#ff7f0e',
'#2ca02c',
'#d62728',
'#9467bd',
'#8c564b',
'#e377c2'
];
currentChart = {
svg,
legend,
series,
buckets,
minX,
maxX,
minY,
maxY,
fill,
colors,
height,
crosshairLine,
crosshairDots,
seriesEls: {},
bucketPixels: [],
xScale: null,
yScale: null,
selected: null,
frozen: false
};
const intervals = [
{unit: 'second', step: 1, ms: 1000},
{unit: 'second', step: 2, ms: 2000},
{unit: 'second', step: 5, ms: 5000},
{unit: 'second', step: 10, ms: 10000},
{unit: 'second', step: 15, ms: 15000},
{unit: 'second', step: 30, ms: 30000},
{unit: 'minute', step: 1, ms: 60000},
{unit: 'minute', step: 2, ms: 120000},
{unit: 'minute', step: 5, ms: 300000},
{unit: 'minute', step: 10, ms: 600000},
{unit: 'minute', step: 15, ms: 900000},
{unit: 'minute', step: 30, ms: 1800000},
{unit: 'hour', step: 1, ms: 3600000},
{unit: 'hour', step: 2, ms: 7200000},
{unit: 'hour', step: 3, ms: 10800000},
{unit: 'hour', step: 4, ms: 14400000},
{unit: 'hour', step: 6, ms: 21600000},
{unit: 'hour', step: 12, ms: 43200000},
{unit: 'day', step: 1, ms: 86400000},
{unit: 'day', step: 2, ms: 172800000},
{unit: 'week', step: 1, ms: 604800000},
{unit: 'week', step: 2, ms: 1209600000},
{unit: 'month', step: 1},
{unit: 'month', step: 3},
{unit: 'month', step: 6},
{unit: 'year', step: 1},
{unit: 'year', step: 2},
{unit: 'year', step: 5},
{unit: 'year', step: 10}
];
function chooseInterval(start, end) {
const span = end - start;
function approxMs(i) {
if (i.ms) return i.ms;
if (i.unit === 'month') return i.step * 2629800000;
if (i.unit === 'year') return i.step * 31557600000;
return 1000;
}
let best = intervals[0];
let bestScore = Infinity;
intervals.forEach(i => {
const count = span / approxMs(i);
const score = Math.abs(count - 15);
if (score < bestScore) {
best = i;
bestScore = score;
}
});
return best;
}
function generateTicks(start, end, intv) {
const ticks = [];
if (intv.unit === 'month' || intv.unit === 'year') {
let d = new Date(start);
d.setUTCDate(1);
if (intv.unit === 'year') d.setUTCMonth(0);
let unitVal =
intv.unit === 'month'
? d.getUTCFullYear() * 12 + d.getUTCMonth()
: d.getUTCFullYear();
unitVal = Math.ceil(unitVal / intv.step) * intv.step;
while (true) {
const year =
intv.unit === 'month' ? Math.floor(unitVal / 12) : unitVal;
const month = intv.unit === 'month' ? unitVal % 12 : 0;
const t = Date.UTC(year, month, 1);
if (t > end) break;
if (t >= start) ticks.push(t);
unitVal += intv.step;
}
} else {
const step = intv.ms * intv.step;
let t = Math.ceil(start / step) * step;
if (intv.unit === 'week') {
const d = new Date(t);
const adj = (d.getUTCDay() + 6) % 7;
t = d.getTime() - adj * 86400000;
(scubaduck) root@95eefccee427:/workspace/scubaduck#I’ll search for the selectedColumns variable to find where it’s defined.
(scubaduck) root@95eefccee427:/workspace/scubaduck# grep -n "selectedColumns" -n
scubaduck/static/js/timeseries_chart.js
38: const valueCols = selectedColumns.slice(groups.length + hasHits);
(scubaduck) root@95eefccee427:/workspace/scubaduck#I think selectedColumns might be defined in table_view.js or somewhere else in the code.
(scubaduck) root@95eefccee427:/workspace/scubaduck# grep -n "selectedColumns" -r
scubaduck/static/js | head
scubaduck/static/js/timeseries_chart.js:38: const valueCols = selectedColumns.s
lice(groups.length + hasHits);
scubaduck/static/js/table_view.js:11: let hitsIndex = selectedColumns.indexOf("
Hits");
scubaduck/static/js/table_view.js:17: selectedColumns.forEach((col, i) => {
scubaduck/static/js/table_view.js:51: const col = selectedColumns[i];
scubaduck/static/js/view_settings.js:15:let selectedColumns = [];
scubaduck/static/js/view_settings.js:447: selectedColumns = groupBy.chips.sli
ce();
scubaduck/static/js/view_settings.js:448: if (document.getElementById('show_h
its').checked) selectedColumns.push('Hits');
scubaduck/static/js/view_settings.js:452: if (!selectedColumns.includes(c
)) selectedColumns.push(c);
scubaduck/static/js/view_settings.js:455: if (dc.include && !selectedColu
mns.includes(dc.name)) selectedColumns.push(dc.name);
scubaduck/static/js/view_settings.js:459: selectedColumns = base.slice();
(scubaduck) root@95eefccee427:/workspace/scubaduck#I'll open view_settings.js to check how selectedColumns is being constructed.
(scubaduck) root@95eefccee427:/workspace/scubaduck# sed -n '410,470p' scubaduck/
static/js/view_settings.js
btn.addEventListener('click', () => {
const dd = document.getElementById(btn.dataset.target);
const show = dd.style.display === 'none' || dd.style.display === '';
document.querySelectorAll('.rel-dropdown').forEach(d => (d.style.display = '
none'));
dd.style.display = show ? 'block' : 'none';
});
});
document.querySelectorAll('.rel-dropdown div').forEach(opt => {
opt.addEventListener('click', () => {
const box = opt.closest('.rel-box');
const input = box.querySelector('input');
input.value = opt.dataset.value || opt.textContent;
opt.parentElement.style.display = 'none';
});
});
document.addEventListener('click', e => {
document.querySelectorAll('.rel-dropdown').forEach(dd => {
if (!dd.parentElement.contains(e.target)) dd.style.display = 'none';
});
});
function updateColumnsTabCount() {
const baseCount = document.querySelectorAll('#column_groups input:checked').le
ngth;
const derivedCount = document.querySelectorAll('#derived_list .derived .d-use:
checked').length;
const btn = document.getElementById('columns_tab');
if (btn) btn.textContent = `Columns (${baseCount + derivedCount})`;
}
function updateSelectedColumns(type = graphTypeSel.value) {
const base = allColumns.filter(name => {
const cb = document.querySelector(`#column_groups input[value="${name}"]`);
if (!cb || !cb.checked) return false;
if (type === 'table' && isStringColumn(name)) return false;
return true;
});
if (type === 'table' || type === 'timeseries') {
selectedColumns = groupBy.chips.slice();
if (document.getElementById('show_hits').checked) selectedColumns.push('Hits
');
const agg = document.getElementById('aggregate').value.toLowerCase();
if (!(type === 'table' && agg === 'count')) {
base.forEach(c => {
if (!selectedColumns.includes(c)) selectedColumns.push(c);
});
derivedColumns.forEach(dc => {
if (dc.include && !selectedColumns.includes(dc.name)) selectedColumns.pu
sh(dc.name);
});
}
} else {
selectedColumns = base.slice();
derivedColumns.forEach(dc => {
if (dc.include) selectedColumns.push(dc.name);
});
}
columnValues[type] = selectedColumns.slice();
const orderCol = document.getElementById('order_by').value;
if (orderCol && !selectedColumns.includes(orderCol)) {
selectedColumns.push(orderCol);
}
updateColumnsTabCount();
}
(scubaduck) root@95eefccee427:/workspace/scubaduck#I'm thinking about how selected columns are handled for 'timeseries' vs 'table' views. For timeseries, base columns and derived columns are included, except when aggregate is 'Count'. For the 'table' view, base columns aren't added for Count, but they are for timeseries, even with Count aggregate. If no columns are selected, only groupBy chips and Hits (if selected) remain. If no columns and show_hits is false, selectedColumns ends up empty, and no value columns are shown on the chart. The issue arises because 'Count' isn't automatically included, though it's part of the query by default.
I’m noticing the 'Count' column isn’t automatically included for aggregation, causing blank charts. It seems when no base columns are selected, 'Count' should still be present, but it's not. I need to check how table_view.js handles 'Count' and adjust accordingly.
(scubaduck) root@95eefccee427:/workspace/scubaduck# sed -n '1,120p' scubaduck/st
atic/js/table_view.js
// Logic for rendering the table based views. Extracted from index.html so that
// the inline script only handles wiring up the UI.
let originalRows = [];
let sortState = { index: null, dir: null };
function renderTable(rows) {
const table = document.getElementById("results");
table.innerHTML = "";
if (rows.length === 0) return;
let hitsIndex = selectedColumns.indexOf("Hits");
let totalHits = 0;
if (hitsIndex !== -1) {
totalHits = rows.reduce((s, r) => s + Number(r[hitsIndex]), 0);
}
const header = document.createElement("tr");
selectedColumns.forEach((col, i) => {
const th = document.createElement("th");
let label = col;
if (
displayType === "table" &&
col !== "Hits" &&
!(groupBy.chips || []).includes(col)
) {
const agg = document.getElementById("aggregate").value.toLowerCase();
label += ` (${agg})`;
}
th.textContent = label;
th.dataset.index = i;
th.addEventListener("click", handleSort);
if (sortState.index === i) {
th.classList.add("sorted");
th.textContent = label + (sortState.dir === "desc" ? " \u25BC" : " \u25B2"
);
}
th.style.textAlign = "left";
header.appendChild(th);
});
table.appendChild(header);
rows.forEach((row) => {
const tr = document.createElement("tr");
tr.addEventListener("click", () => {
const wasSelected = tr.classList.contains("selected");
document
.querySelectorAll("#results tr.selected")
.forEach((el) => el.classList.remove("selected"));
if (!wasSelected) {
tr.classList.add("selected");
}
});
row.forEach((v, i) => {
const col = selectedColumns[i];
const td = document.createElement("td");
if (isTimeColumn(col)) {
let d;
const t = (columnTypes[col] || "").toUpperCase();
if (t.includes("TIMESTAMP") || t.includes("DATE") || t.includes("TIME"))
{
d = new Date(v);
} else {
const unit = document.getElementById("time_unit").value;
const factors = { s: 1000, ms: 1, us: 0.001, ns: 0.000001 };
d = new Date(Number(v) * (factors[unit] || 1000));
}
td.textContent = d.toLocaleString("en-US", {
weekday: "short",
month: "short",
day: "numeric",
year: "numeric",
hour: "numeric",
minute: "numeric",
second: "numeric",
hour12: true,
timeZoneName: "short",
});
td.classList.add("date");
} else {
if (col === "Hits") {
const pct = totalHits ? ((v / totalHits) * 100).toFixed(1) : "0";
td.textContent = `${formatNumber(v)} (${pct}%)`;
} else {
td.textContent = isStringColumn(col) ? v : formatNumber(v);
}
}
if (!isStringColumn(col) && !isTimeColumn(col)) {
td.classList.add("numeric");
}
td.style.textAlign = isStringColumn(col) ? "left" : "right";
tr.appendChild(td);
});
table.appendChild(tr);
});
// ensure table does not overflow unless necessary
const view = document.getElementById("view");
if (table.scrollWidth <= view.clientWidth) {
table.style.width = "100%";
}
}
function handleSort(e) {
const idx = parseInt(e.target.dataset.index, 10);
if (sortState.index !== idx) {
sortState.index = idx;
sortState.dir = "asc";
} else if (sortState.dir === "asc") {
sortState.dir = "desc";
} else if (sortState.dir === "desc") {
sortState.index = null;
sortState.dir = null;
} else {
sortState.dir = "asc";
}
let rows = originalRows.slice();
if (sortState.index !== null) {
rows.sort((a, b) => {
const va = a[sortState.index];
const vb = b[sortState.index];
if (va === vb) return 0;
if (sortState.dir === "desc") return va < vb ? 1 : -1;
return va > vb ? 1 : -1;
});
}
(scubaduck) root@95eefccee427:/workspace/scubaduck#I'm reviewing how 'Count' and Hits, along with base columns, are handled for both timeseries and table views. Looks like timeseries includes base columns regardless of aggregator. For table, even with a 'count' aggregator, it doesn't automatically add 'Count', which could be an issue. Let me check the relevant tests to confirm.
(scubaduck) root@95eefccee427:/workspace/scubaduck# grep -n "test_timeseries_cou
nt_no_columns" -n tests/test_server_timeseries.py
109:def test_timeseries_count_no_columns() -> None:_server_timeseries.py
data = rv.get_json()
assert rv.status_code == 200
rows = data["rows"]
from dateutil import parser
timestamps = [parser.parse(r[0]).replace(tzinfo=None) for r in rows]
assert timestamps == sorted(timestamps)
def test_timeseries_count_no_columns() -> None:
app = server.app
client = app.test_client()
payload: dict[str, Any] = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "timeseries",
"granularity": "1 day",
"columns": [],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
rows = data["rows"]
assert len(rows) == 2
assert rows[0][1] == 2
assert rows[1][1] == 2
assert len(rows[0]) == 3
def test_timeseries_limit_applies_to_series() -> None:
app = server.app
client = app.test_client()
payload: dict[str, Any] = {
"table": "events",
(scubaduck) root@95eefccee427:/workspace/scubaduck#Tests expect 3 columns: bucket, Hits, Count. Server seems to work, but frontend may have an issue.
The UI might not include 'Count' in selectedColumns. I'll check where updateSelectedColumns is used.
(scubaduck) root@95eefccee427:/workspace/scubaduck# grep -n "updateSelectedColum
ns" -r scubaduck/static/js | head
scubaduck/static/js/view_settings.js:144: updateSelectedColumns(prevType);
scubaduck/static/js/view_settings.js:175: updateSelectedColumns(newType);
scubaduck/static/js/view_settings.js:316: cb.addEventListener('change', u
pdateSelectedColumns);
scubaduck/static/js/view_settings.js:325: updateSelectedColumns();
scubaduck/static/js/view_settings.js:330: updateSelectedColumns();
scubaduck/static/js/view_settings.js:338: updateSelectedColumns();
scubaduck/static/js/view_settings.js:343: updateSelectedColumns();
scubaduck/static/js/view_settings.js:345: updateSelectedColumns();
scubaduck/static/js/view_settings.js:439:function updateSelectedColumns(type = g
raphTypeSel.value) {
scubaduck/static/js/view_settings.js:655: updateSelectedColumns();
(scubaduck) root@95eefccee427:/workspace/scubaduck#I'll dig deeper around line 620 to better understand how the request is being built.
static/js/view_settings.js
});
derivedColumns.push(obj);
refreshDerivedColumns();
}
function removeDerived(btn) {
const el = btn.closest('.derived');
const idx = derivedColumns.findIndex(d => d.el === el);
if (idx !== -1) {
derivedColumns.splice(idx, 1);
}
el.remove();
refreshDerivedColumns();
}
function refreshDerivedColumns() {
allColumns.splice(0, allColumns.length, ...baseColumns);
stringColumns.splice(0, stringColumns.length, ...baseStringColumns);
integerColumns.splice(0, integerColumns.length, ...baseIntegerColumns);
timeColumns.splice(0, timeColumns.length, ...baseTimeColumns);
timeColumnOptions.splice(0, timeColumnOptions.length, ...baseTimeColumnOptions
);
derivedColumns.forEach(d => {
d.type = d.el.querySelector('.d-type').value;
d.name = d.el.querySelector('.d-name').value;
d.expr = d.el.querySelector('.d-expr').value;
d.include = d.el.querySelector('.d-use').checked;
allColumns.push(d.name);
columnTypes[d.name] = d.type;
if (d.type === 'string') {
stringColumns.push(d.name);
} else {
integerColumns.push(d.name);
timeColumnOptions.push(d.name);
}
});
updateSelectedColumns();
}
let lastQueryTime = 0;
let queryStart = 0;
function dive(push=true) {
const params = collectParams();
if (push) {
history.pushState(params, '', paramsToSearch(params));
}
const payload = Object.assign({}, params);
const dcMap = {};
(params.derived_columns || []).forEach(d => {
if (d.include) dcMap[d.name] = d.expr;
});
payload.derived_columns = dcMap;
const view = document.getElementById('view');
view.innerHTML = '<p>Loading...</p>';
window.lastResults = undefined;
queryStart = performance.now();
fetch('/api/query', {method:'POST', headers:{'Content-Type':'application/json'
}, body:JSON.stringify(payload)})
.then(async r => {
const data = await r.json();
if (!r.ok) throw data;
return data;(scubaduck) root@95eefccee427:/workspace/scubaduck# sed -n '680,760p' scubaduck/
static/js/view_settings.js
return data;
})
.then(data => {
lastQueryTime = Math.round(performance.now() - queryStart);
showResults(data);
})
.catch(err => {
showError(err);
});
}
function collectParams() {
updateSelectedColumns();
const payload = {
table: document.getElementById('table').value,
time_column: document.getElementById('time_column').value,
time_unit: document.getElementById('time_unit').value,
start: document.getElementById('start').value,
end: document.getElementById('end').value,
order_by: document.getElementById('order_by').value,
order_dir: orderDir,
limit: parseInt(document.getElementById('limit').value, 10),
columns: selectedColumns.filter(c =>
c !== 'Hits' && !derivedColumns.some(dc => dc.name === c)
),
samples_columns: columnValues.samples.slice(),
table_columns: columnValues.table.slice(),
timeseries_columns: columnValues.timeseries.slice(),
graph_type: graphTypeSel.value,
filters: Array.from(document.querySelectorAll('#filters .filter')).map(f =>
{
const chips = f.chips || [];
const op = f.querySelector('.f-op').value;
let value = null;
if (op !== 'empty' && op !== '!empty') {
value = chips.length === 0 ? null : (chips.length === 1 ? chips[0] : chi
ps);
}
const colSel = f.querySelector('.f-col select') || f.querySelector('.f-col
');
return {column: colSel.value, op, value};
}),
derived_columns: Array.from(document.querySelectorAll('#derived_list .derive
d')).map(d => ({
type: d.querySelector('.d-type').value,
name: d.querySelector('.d-name').value,
expr: d.querySelector('.d-expr').value,
include: d.querySelector('.d-use').checked,
}))
};
if (graphTypeSel.value === 'table' || graphTypeSel.value === 'timeseries') {
payload.group_by = groupBy.chips || [];
payload.aggregate = document.getElementById('aggregate').value;
payload.show_hits = document.getElementById('show_hits').checked;
}
if (graphTypeSel.value === 'timeseries') {
const xval = document.getElementById('x_axis').value;
if (xval) payload.x_axis = xval;
payload.granularity = document.getElementById('granularity').value;
payload.fill = document.getElementById('fill').value;
}
return payload;
}
function paramsToSearch(params) {
const sp = new URLSearchParams();
if (params.table) sp.set('table', params.table);
if (params.time_column) sp.set('time_column', params.time_column);
if (params.time_unit) sp.set('time_unit', params.time_unit);
if (params.start) sp.set('start', params.start);
if (params.end) sp.set('end', params.end);
if (params.order_by) sp.set('order_by', params.order_by);
if (params.order_dir) sp.set('order_dir', params.order_dir);
if (params.limit !== null && params.limit !== undefined) sp.set('limit', param
s.limit);
if (params.samples_columns && params.samples_columns.length) sp.set('samples_c
olumns', params.samples_columns.join(','));
if (params.table_columns && params.table_columns.length) sp.set('table_columns
', params.table_columns.join(','));
if (params.timeseries_columns && params.timeseries_columns.length) sp.set('tim
eseries_columns', params.timeseries_columns.join(','));
if (params.filters && params.filters.length) sp.set('filters', JSON.stringify(
params.filters));
if (params.derived_columns && params.derived_columns.length) sp.set('derived_c
olumns', JSON.stringify(params.derived_columns));
if (params.graph_type) sp.set('graph_type', params.graph_type);
if (params.graph_type === 'table' || params.graph_type === 'timeseries') {
if (params.group_by && params.group_by.length) sp.set('group_by', params.gro
up_by.join(','));
if (params.aggregate) sp.set('aggregate', params.aggregate);
if (params.show_hits) sp.set('show_hits', '1');
}
(scubaduck) root@95eefccee427:/workspace/scubaduck#I'll search for 'Count' or 'aggregate' inside updateSelectedColumns to clarify this behavior.
(scubaduck) root@95eefccee427:/workspace/scubaduck# grep -n "aggregate" -n scuba
duck/static/js/view_settings.js | head
149: document.getElementById('aggregate_field').style.display = showTable || sh
owTS ? 'flex' : 'none';
354: initDropdown(document.getElementById('aggregate'));
449: const agg = document.getElementById('aggregate').value.toLowerCase();
596: <option value="aggregated">Aggregated</option>
728: payload.aggregate = document.getElementById('aggregate').value;
758: if (params.aggregate) sp.set('aggregate', params.aggregate);
801: if (params.aggregate) document.getElementById('aggregate').value = params.
aggregate;
848: document.getElementById('aggregate').value = 'Count';
891: if (sp.has('aggregate')) params.aggregate = sp.get('aggregate');
(scubaduck) root@95eefccee427:/workspace/scubaduck#I'll check lines 430-460 for where updateSelectedColumns is defined.
(scubaduck) root@95eefccee427:/workspace/scubaduck# sed -n '440,466p' scubaduck/
static/js/view_settings.js
const base = allColumns.filter(name => {
const cb = document.querySelector(`#column_groups input[value="${name}"]`);
if (!cb || !cb.checked) return false;
if (type === 'table' && isStringColumn(name)) return false;
return true;
});
if (type === 'table' || type === 'timeseries') {
selectedColumns = groupBy.chips.slice();
if (document.getElementById('show_hits').checked) selectedColumns.push('Hits
');
const agg = document.getElementById('aggregate').value.toLowerCase();
if (!(type === 'table' && agg === 'count')) {
base.forEach(c => {
if (!selectedColumns.includes(c)) selectedColumns.push(c);
});
derivedColumns.forEach(dc => {
if (dc.include && !selectedColumns.includes(dc.name)) selectedColumns.pu
sh(dc.name);
});
}
} else {
selectedColumns = base.slice();
derivedColumns.forEach(dc => {
if (dc.include) selectedColumns.push(dc.name);
});
}
columnValues[type] = selectedColumns.slice();
const orderCol = document.getElementById('order_by').value;
if (orderCol && !selectedColumns.includes(orderCol)) {
(scubaduck) root@95eefccee427:/workspace/scubaduck#I'll continue to scroll from line 470 to 500.
(scubaduck) root@95eefccee427:/workspace/scubaduck# sed -n '466,520p' scubaduck/
static/js/view_settings.js
if (orderCol && !selectedColumns.includes(orderCol)) {
selectedColumns.push(orderCol);
}
updateColumnsTabCount();
}
function isStringColumn(name) {
const t = (columnTypes[name] || '').toUpperCase();
return t.includes('CHAR') || t.includes('STRING') || t.includes('VARCHAR');
}
function isIntegerColumn(name) {
const t = (columnTypes[name] || '').toUpperCase();
return t.includes('INT');
}
function isTimeColumn(name) {
const t = (columnTypes[name] || '').toUpperCase();
if (t.includes('TIMESTAMP') || t.includes('DATE') || t.includes('TIME')) retur
n true;
const sel = document.getElementById('time_column').value;
const xsel = document.getElementById('x_axis').value;
if (name === sel || name === xsel) return true;
return false;
}
function formatNumber(val) {
if (typeof val !== 'number') val = Number(val);
if (Number.isNaN(val)) return '';
if (val === 0) return '0';
const abs = Math.abs(val);
if (abs > 999.999) {
const units = [
{n: 1e12, s: 'T'},
{n: 1e9, s: 'B'},
{n: 1e6, s: 'M'},
{n: 1e3, s: 'K'},
];
for (const u of units) {
if (abs >= u.n) {
return (val / u.n).toFixed(2) + ' ' + u.s;
}
}
}
if (abs < 0.0005) return '0.000';
if (Number.isInteger(val)) return val.toString();
return val.toFixed(3);
}
function addFilter() {
const container = document.createElement('div');
container.className = 'filter';
container.innerHTML = `
<div class="filter-row">
<select class="f-col"></select>
(scubaduck) root@95eefccee427:/workspace/scubaduck#I’m fixing the issue where 'Count' isn't included when it's supposed to—especially for 'timeseries'.
(scubaduck) root@95eefccee427:/workspace/scubaduck# grep -n "table_count_no_colu
mns" -r tests
tests/test_web_table.py:333:def test_table_count_no_columns(page: Any, test_data
set_server_url: str) -> None:
(scubaduck) root@95eefccee427:/workspace/scubaduck#I'll review the test to understand its behavior.
def test_sql_query_display(page: Any, server_url: str) -> None:
data = run_query(
page,
server_url,
start="2024-01-01 00:00:00",
end="2024-01-02 00:00:00",
order_by="timestamp",
limit=10,
)
sql = data["sql"]
displayed = page.text_content("#sql_query")
assert displayed is not None
assert displayed.strip() == sql
def test_table_count_no_columns(page: Any, test_dataset_server_url: str) -> None
:
page.goto(test_dataset_server_url)
page.wait_for_selector("#order_by option", state="attached")
select_value(page, "#graph_type", "table")
page.click("text=Columns")
page.click("#columns_all")
page.click("text=View Settings")
page.evaluate("groupBy.chips = ['id']; groupBy.renderChips();")
select_value(page, "#aggregate", "Count")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
headers = page.locator("#results th").all_inner_texts()
assert headers == ["id", "Hits"]
col_count = page.locator("#results th").count()
row_count = page.locator("#results tr").count()
assert col_count == 2
assert row_count == 3
overflow = page.evaluate(
"var v=document.getElementById('view'); v.scrollWidth > v.clientWidth"
)
assert not overflow
(scubaduck) root@95eefccee427:/workspace/scubaduck#There’s a test validating that Count aggregators hide the "Count" column in table view. Now, I’ll check tests for timeseries charts in tests/test_web_timeseries.py.
(scubaduck) root@95eefccee427:/workspace/scubaduck# ls tests | grep web_time
test_web_timeseries.pyfrom tests.web_utils import select_value
def test_timeseries_default_query(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#graph_type", state="attached")
select_value(page, "#graph_type", "timeseries")
page.click("text=Columns")
page.check("#column_groups input[value='value']")
page.click("text=View Settings")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
data = page.evaluate("window.lastResults")
assert "error" not in data
assert page.is_visible("#chart")
page.click("text=Columns")
assert not page.is_checked("#column_groups input[value='timestamp']")
def test_timeseries_single_bucket(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#graph_type", state="attached")
page.fill("#start", "2024-01-01 00:00:00")
page.fill("#end", "2024-01-01 00:00:00")
select_value(page, "#graph_type", "timeseries")
page.click("text=Columns")
page.check("#column_groups input[value='value']")
page.click("text=View Settings")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
path = page.get_attribute("#chart path", "d")
assert path is not None and "NaN" not in path
def test_timeseries_fill_options(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#graph_type", state="attached")
page.fill("#start", "2024-01-01 00:00:00")
page.fill("#end", "2024-01-02 03:00:00")
select_value(page, "#graph_type", "timeseries")
page.click("text=Columns")
page.check("#column_groups input[value='value']")
page.click("text=View Settings")
select_value(page, "#granularity", "1 hour")
select_value(page, "#fill", "0")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
path_zero = page.get_attribute("#chart path", "d")
assert path_zero is not None and path_zero.count("L") > 20
select_value(page, "#fill", "connect")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
path_conn = page.get_attribute("#chart path", "d")
assert path_conn is not None and path_conn.count("M") == 1
select_value(page, "#fill", "blank")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
path_blank = page.get_attribute("#chart path", "d")
assert path_blank is not None and path_blank.count("M") > 1
def test_timeseries_hover_highlight(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#graph_type", state="attached")
select_value(page, "#graph_type", "timeseries")
page.click("text=Columns")
page.check("#column_groups input[value='value']")
page.click("text=View Settings")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
page.wait_for_selector("#chart path", state="attached")
path_el = page.query_selector("#chart path")
assert path_el
page.evaluate(
"el => el.dispatchEvent(new MouseEvent('mouseenter', {bubbles: true}))",
path_el,
)
width = page.evaluate(
"getComputedStyle(document.querySelector('#chart path')).strokeWidth"
)
assert "2.5" in width
color = page.evaluate(
"getComputedStyle(document.querySelector('#legend .legend-item')).backgr
oundColor"
)
assert "221, 221, 221" in color
def test_timeseries_crosshair(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#graph_type", state="attached")
select_value(page, "#graph_type", "timeseries")
page.click("text=Columns")
page.check("#column_groups input[value='value']")
page.click("text=View Settings")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
page.wait_for_selector("#chart path", state="attached")
page.eval_on_selector(
"#chart",
"el => { const r = el.getBoundingClientRect(); el.dispatchEvent(new Mous
eEvent('mousemove', {clientX: r.left + r.width/2, clientY: r.top + r.height/2, b
ubbles: true})); }",
)
line_display = page.evaluate(
"document.getElementById('crosshair_line').style.display"
)
assert line_display != "none"
count = page.eval_on_selector_all("#crosshair_dots circle", "els => els.leng
th")
assert count > 0
page.eval_on_selector(
"#chart",
"el => el.dispatchEvent(new MouseEvent('mouseleave', {bubbles: true}))",
)
line_display = page.evaluate(
"document.getElementById('crosshair_line').style.display"
)
assert line_display == "none"
def test_timeseries_crosshair_freeze(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#graph_type", state="attached")
select_value(page, "#graph_type", "timeseries")
page.click("text=Columns")
page.check("#column_groups input[value='value']")
page.click("text=View Settings")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
page.wait_for_selector("#chart path", state="attached")
page.eval_on_selector(
"#chart",
"el => { const r = el.getBoundingClientRect(); el.dispatchEvent(new Mous
eEvent('mousemove', {clientX: r.left + r.width/2, clientY: r.top + r.height/2, b
ubbles: true})); }",
)
page.eval_on_selector(
"#chart",
"el => { const r = el.getBoundingClientRect(); el.dispatchEvent(new Mous
eEvent('click', {clientX: r.left + r.width/2, clientY: r.top + r.height/2, bubbl
es: true})); }",
)
line_display = page.evaluate(
"document.getElementById('crosshair_line').style.display"
)
assert line_display != "none"
pos1 = page.evaluate("document.getElementById('crosshair_line').getAttribute
('x1')")
page.eval_on_selector(
"#chart",
"el => { const r = el.getBoundingClientRect(); el.dispatchEvent(new Mous
eEvent('mousemove', {clientX: r.left + r.width/4, clientY: r.top + r.height/2, b
ubbles: true})); }",
)
pos2 = page.evaluate("document.getElementById('crosshair_line').getAttribute
('x1')")
(scubaduck) root@95eefccee427:/workspace/scubaduck#I'll continue by loading additional lines to check for tests.
"#chart",
"el => el.dispatchEvent(new MouseEvent('mouseleave', {bubbles: true}))",
)
line_display = page.evaluate(
"document.getElementById('crosshair_line').style.display"
)
assert line_display != "none"
page.eval_on_selector(
"#chart",
"el => { const r = el.getBoundingClientRect(); el.dispatchEvent(new Mous
eEvent('click', {clientX: r.left + r.width/2, clientY: r.top + r.height/2, bubbl
es: true})); }",
)
line_display = page.evaluate(
"document.getElementById('crosshair_line').style.display"
)
assert line_display == "none"
def test_timeseries_auto_timezone(browser: Any, server_url: str) -> None:
context = browser.new_context(timezone_id="America/New_York")
page = context.new_page()
page.goto(server_url)
page.wait_for_selector("#graph_type", state="attached")
select_value(page, "#graph_type", "timeseries")
page.click("text=Columns")
page.check("#column_groups input[value='value']")
page.click("text=View Settings")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
path = page.get_attribute("#chart path", "d")
context.close()
assert path is not None
coords = [float(p.split(" ")[1]) for p in path.replace("M", "L").split("L")[
1:]]
assert max(coords) > min(coords)
def test_timeseries_multi_series(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#graph_type", state="attached")
select_value(page, "#graph_type", "timeseries")
page.click("text=Columns")
page.check("#column_groups input[value='value']")
page.click("text=Add Derived")
expr = page.query_selector("#derived_list .derived textarea")
assert expr
name_inp = page.query_selector("#derived_list .derived .d-name")
assert name_inp
name_inp.fill("value_2")
expr.fill("value * 2")
page.click("text=View Settings")
page.fill("#start", "2024-01-01 00:00:00")
page.fill("#end", "2024-01-03 00:00:00")
select_value(page, "#granularity", "1 hour")
select_value(page, "#aggregate", "Avg")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
count = page.eval_on_selector_all("#chart path", "els => els.length")
assert count == 2
def test_timeseries_resize(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#graph_type", state="attached")
select_value(page, "#graph_type", "timeseries")
page.click("text=Columns")
page.check("#column_groups input[value='value']")
page.click("text=View Settings")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
page.wait_for_selector("#chart path", state="attached")
def chart_info() -> dict[str, float]:
return page.evaluate(
"() => {const p=document.querySelector('#chart path'); const nums=p.
getAttribute('d').match(/[-0-9.]+/g).map(parseFloat); return {width: parseFloat(
document.getElementById('chart').getAttribute('width')), last: nums[nums.length-
2]};}"
)
before = chart_info()
legend_width = page.evaluate(
"parseFloat(getComputedStyle(document.getElementById('legend')).width)"
)
assert page.evaluate(
"() => document.getElementById('legend').getBoundingClientRect().right <
= document.getElementById('chart').getBoundingClientRect().left"
)
page.evaluate("document.getElementById('sidebar').style.width='200px'")
page.wait_for_function(
"width => document.getElementById('chart').getAttribute('width') != widt
h",
arg=before["width"],
)
after = chart_info()
legend_width_after = page.evaluate(
"parseFloat(getComputedStyle(document.getElementById('legend')).width)"
)
assert after["width"] > before["width"]
assert after["last"] > before["last"]
assert legend_width_after == legend_width
def test_timeseries_no_overflow(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#graph_type", state="attached")
select_value(page, "#graph_type", "timeseries")
page.click("text=Columns")
page.check("#column_groups input[value='value']")
page.click("text=View Settings")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
overflow = page.evaluate(
"var v=document.getElementById('view'); v.scrollWidth > v.clientWidth"
)
assert not overflow
def test_timeseries_axis_ticks(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#graph_type", state="attached")
select_value(page, "#graph_type", "timeseries")
page.click("text=Columns")
page.check("#column_groups input[value='value']")
page.click("text=View Settings")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
page.wait_for_selector("#chart text.tick-label", state="attached")
count = page.eval_on_selector_all("#chart text.tick-label", "els => els.leng
th")
assert count > 2
def test_timeseries_y_axis_labels(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#graph_type", state="attached")
select_value(page, "#graph_type", "timeseries")
page.click("text=Columns")
page.check("#column_groups input[value='value']")
page.click("text=View Settings")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
page.wait_for_selector("#chart text.y-tick-label", state="attached")
count = page.eval_on_selector_all("#chart text.y-tick-label", "els => els.le
ngth")
grid_count = page.eval_on_selector_all("#chart line.grid", "els => els.lengt
h")
assert count > 0 and count == grid_count
def test_timeseries_interval_offset(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#graph_type", state="attached")
select_value(page, "#graph_type", "timeseries")
page.click("text=Columns")
page.check("#column_groups input[value='value']")
page.click("text=View Settings")
page.fill("#start", "2024-01-01 00:00:00")
page.fill("#end", "2024-01-03 12:00:00")
select_value(page, "#granularity", "1 hour")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
(scubaduck) root@95eefccee427:/workspace/scubaduck#I'm scrolling past line 320 to reach additional content.
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
page.wait_for_selector("#chart text.tick-label", state="attached")
labels = page.eval_on_selector_all(
"#chart text.tick-label", "els => els.map(e => e.textContent)"
)
assert labels
assert all(lbl != "00:00" for lbl in labels)
times = [lbl for lbl in labels if ":" in lbl]
assert times
for t in times:
h = int(t.split(":")[0])
assert h % 4 == 0
def test_timeseries_legend_values(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#graph_type", state="attached")
select_value(page, "#graph_type", "timeseries")
page.click("text=Columns")
page.check("#column_groups input[value='value']")
page.click("text=View Settings")
page.evaluate("g => { groupBy.chips = g; groupBy.renderChips(); }", ["user"]
)
select_value(page, "#aggregate", "Avg")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
headers = page.evaluate(
"() => Array.from(document.querySelectorAll('#legend .legend-header')).m
ap(e => e.textContent)"
)
assert any(h.startswith("alice") for h in headers)
page.wait_for_selector("#chart path", state="attached")
page.eval_on_selector(
"#chart",
"el => { const r=el.getBoundingClientRect(); el.dispatchEvent(new MouseE
vent('mousemove', {clientX:r.left+r.width/2, clientY:r.top+r.height/2, bubbles:t
rue})); }",
)
value = page.evaluate("document.querySelector('#legend .legend-value').textC
ontent")
assert value != ""
def test_timeseries_group_links(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#graph_type", state="attached")
select_value(page, "#graph_type", "timeseries")
page.click("text=Columns")
page.check("#column_groups input[value='value']")
page.click("text=View Settings")
page.fill("#start", "2024-01-01 00:00:00")
page.fill("#end", "2024-01-02 03:00:00")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
assert page.text_content("#legend .drill-links h4") == "Group by"
page.click("#legend .drill-links a:text('user')")
page.wait_for_function("window.lastResults !== undefined")
chips = page.evaluate("groupBy.chips")
assert chips == ["user"]
assert page.text_content("#legend .drill-links h4") == "Drill up"
assert page.is_visible("#legend .drill-links a:text('Aggregate')")
(scubaduck) root@95eefccee427:/workspace/scubaduck#I'll search for "web_timeseries.*Count" and related patterns to locate the tests.
(scubaduck) root@95eefccee427:/workspace/scubaduck# grep -n "Count" -n tests/tes
t_web_timeseries.py | head
(scubaduck) root@95eefccee427:/workspace/scubaduck#The bug occurs when 'Count' leads to a blank chart. I'll confirm the issue first.
For timeseries, if 'Count' is selected with no columns, I'll add it to selectedColumns.
For timeseries with 'Count', I'll ensure 'Count' is added to the selected columns, as needed.
The index calculation needs fixing as Hits is mistakenly treated as Count in chart data.
duck/static/js
scubaduck/static/js/timeseries_chart.js:32: const hasHits = document.getElement
ById('show_hits').checked ? 1 : 0;
scubaduck/static/js/table_view.js:128: !document.getElementById("show_hits").
checked;
scubaduck/static/js/view_settings.js:150: document.getElementById('show_hits_fi
eld').style.display = showTable ? 'flex' : 'none';
scubaduck/static/js/view_settings.js:448: if (document.getElementById('show_h
its').checked) selectedColumns.push('Hits');
scubaduck/static/js/view_settings.js:729: payload.show_hits = document.getEle
mentById('show_hits').checked;
scubaduck/static/js/view_settings.js:759: if (params.show_hits) sp.set('show_
hits', '1');
scubaduck/static/js/view_settings.js:802: document.getElementById('show_hits').
checked = params.show_hits ?? true;
scubaduck/static/js/view_settings.js:849: document.getElementById('show_hits').
checked = true;
scubaduck/static/js/view_settings.js:892: if (sp.has('show_hits')) params.show_
hits = sp.get('show_hits') === '1';
(scubaduck) root@95eefccee427:/workspace/scubaduck#I'll check around the line where show_hits is referenced to better understand it.
(scubaduck) root@95eefccee427:/workspace/scubaduck# sed -n '116,140p' scubaduck/
static/js/table_view.js
if (va === vb) return 0;
if (sortState.dir === "desc") return va < vb ? 1 : -1;
return va > vb ? 1 : -1;
});
}
renderTable(rows);
}
function showResults(data) {
window.lastResults = data;
const hideHits =
(graphTypeSel.value === "table" || graphTypeSel.value === "timeseries") &&
!document.getElementById("show_hits").checked;
if (hideHits && data.rows.length) {
const groupCount =
(graphTypeSel.value === "timeseries" ? 1 : 0) +
((groupBy.chips || []).length || 0);
data.rows.forEach((r) => r.splice(groupCount, 1));
}
const view = document.getElementById("view");
if (graphTypeSel.value === "timeseries") {
showTimeSeries(data);
} else {
if (data.rows.length === 0) {
view.innerHTML =(scubaduck) root@95eefccee427:/workspace/scubaduck# sed -n '140,200p' scubaduck/
static/js/table_view.js
view.innerHTML =
'<p id="empty-message">Empty data provided to table</p><table id="result
s"></table>';
} else {
view.innerHTML = '<table id="results"></table>';
}
originalRows = data.rows.slice();
sortState = { index: null, dir: null };
renderTable(originalRows);
}
const sqlEl = document.createElement("pre");
sqlEl.id = "sql_query";
sqlEl.style.whiteSpace = "pre-wrap";
sqlEl.style.marginTop = "10px";
sqlEl.textContent = data.sql;
view.appendChild(sqlEl);
document.getElementById("query_info").textContent = `Your query took about ${l
astQueryTime} ms`;
}
function showError(err) {
window.lastResults = err;
const view = document.getElementById("view");
let msg = "";
if (typeof err === "string") {
msg = err;
} else if (err) {
msg = err.error || "Error";
if (err.sql) {
msg += "\nSQL: " + err.sql;
}
if (err.traceback) {
msg += "\n" + err.traceback;
}
}
view.innerHTML = `<pre id="error-message">${msg}</pre>`;
document.getElementById("query_info").textContent = "";
}
(scubaduck) root@95eefccee427:/workspace/scubaduck#showResults removes the 'hits' column based on show_hits setting, fixing earlier issues.
I'll add a test to verify selecting 'Count' when using timeseries aggregation.
(scubaduck) root@95eefccee427:/workspace/scubaduck# grep -n "test_dataset_server
_url" -r tests
tests/test_web_table.py:16:def test_dataset_server_url() -> Iterator[str]:
tests/test_web_table.py:333:def test_table_count_no_columns(page: Any, test_data
set_server_url: str) -> None:
tests/test_web_table.py:334: page.goto(test_dataset_server_url)
(scubaduck) root@95eefccee427:/workspace/scubaduck#I'll take a look at tests/test_web_table.py to examine the fixture.
from __future__ import annotations
from typing import Any
from collections.abc import Iterator
import threading
import pytest
from werkzeug.serving import make_server
from scubaduck.server import create_app
from tests.web_utils import run_query, select_value
@pytest.fixture()
def test_dataset_server_url() -> Iterator[str]:
app = create_app("TEST")
httpd = make_server("127.0.0.1", 0, app)
port = httpd.server_port
thread = threading.Thread(target=httpd.serve_forever)
thread.start()
try:
yield f"http://127.0.0.1:{port}"
finally:
httpd.shutdown()
thread.join()
def test_table_sorting(page: Any, server_url: str) -> None:
run_query(
page,
server_url,
start="2024-01-01 00:00:00",
end="2024-01-03 00:00:00",
order_by="user",
order_dir="ASC",
limit=100,
)
# header alignment
align = page.evaluate(
"getComputedStyle(document.querySelector('#results th')).textAlign"
)
assert align == "left"
header = page.locator("#results th").nth(3)
def values() -> list[str]:
return page.locator("#results td:nth-child(4)").all_inner_texts()
orig_rows = values()
assert orig_rows == ["alice", "bob", "alice", "charlie"]
first_sql = page.evaluate("window.lastResults.sql")
header.click()
assert values() == sorted(orig_rows)
assert header.inner_text().endswith("▲")
color = page.evaluate(
"getComputedStyle(document.querySelector('#results th:nth-child(4)')).co
lor"
)
(scubaduck) root@95eefccee427:/workspace/scubaduck#I need to inspect the run_query function in web_utils.py to test the bug replication.
from __future__ import annotations
from typing import Any
def select_value(page: Any, selector: str, value: str) -> None:
page.evaluate(
"arg => setSelectValue(arg.sel, arg.val)",
{"sel": selector, "val": value},
)
def run_query(
page: Any,
url: str,
*,
start: str | None = None,
end: str | None = None,
order_by: str | None = None,
order_dir: str | None = "ASC",
limit: int | None = None,
group_by: list[str] | None = None,
aggregate: str | None = None,
) -> dict[str, Any]:
page.goto(url)
page.wait_for_selector("#order_by option", state="attached")
page.wait_for_selector("#order_dir", state="attached")
page.wait_for_function("window.lastResults !== undefined")
if start is not None:
page.fill("#start", start)
if end is not None:
page.fill("#end", end)
if order_by is not None:
select_value(page, "#order_by", order_by)
if order_dir is not None and order_dir == "DESC":
page.click("#order_dir")
if limit is not None:
page.fill("#limit", str(limit))
if group_by is not None:
select_value(page, "#graph_type", "table")
page.evaluate(
"g => { groupBy.chips = g; groupBy.renderChips(); }",
group_by,
)
if aggregate is not None:
select_value(page, "#graph_type", "table")
select_value(page, "#aggregate", aggregate)
if page.input_value("#graph_type") != "samples":
page.click("text=Columns")
page.wait_for_selector("#column_groups input", state="attached")
if not page.is_checked("#column_groups input[value='value']"):
page.check("#column_groups input[value='value']")
order_col = order_by or page.input_value("#order_by")
if order_col and not page.is_checked(
f"#column_groups input[value='{order_col}']"
):
if page.query_selector(f"#column_groups input[value='{order_col}']")
:
page.check(f"#column_groups input[value='{order_col}']")
page.click("text=View Settings")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
return page.evaluate("window.lastResults")
(scubaduck) root@95eefccee427:/workspace/scubaduck#Hmm, looks like run_query defaults to 'table'. Let's see if it's used for timeseries too.
(scubaduck) root@95eefccee427:/workspace/scubaduck# grep -n "run_query(" -r test
s | head
tests/test_web_table.py:30: run_query(
tests/test_web_table.py:78: data = run_query(
tests/test_web_table.py:92: run_query(
tests/test_web_table.py:132: run_query(
tests/test_web_table.py:151: run_query(
tests/test_web_table.py:166: data = run_query(
tests/test_web_table.py:218: data = run_query(
tests/test_web_table.py:233: data = run_query(
tests/test_web_table.py:250: run_query(
tests/test_web_table.py:272: run_query(page, server_url, order_by="timestamp"
, limit=10)
(scubaduck) root@95eefccee427:/workspace/scubaduck#I'll review tests/test_web_timeseries.py to understand how they handle timeseries queries.
(scubaduck) root@95eefccee427:/workspace/scubaduck# grep -n "goto(" -n tests/tes
t_web_timeseries.py | head
9: page.goto(server_url)
26: page.goto(server_url)
42: page.goto(server_url)
75: page.goto(server_url)
102: page.goto(server_url)
133: page.goto(server_url)
183: page.goto(server_url)
200: page.goto(server_url)
225: page.goto(server_url)
263: page.goto(server_url)
(scubaduck) root@95eefccee427:/workspace/scubaduck#I'm opening tests/test_web_timeseries.py to check out the setup.
meseries.py
from __future__ import annotations
from typing import Any
from tests.web_utils import select_value
def test_timeseries_default_query(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#graph_type", state="attached")
select_value(page, "#graph_type", "timeseries")
page.click("text=Columns")
page.check("#column_groups input[value='value']")
page.click("text=View Settings")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
data = page.evaluate("window.lastResults")
assert "error" not in data
assert page.is_visible("#chart")
page.click("text=Columns")
assert not page.is_checked("#column_groups input[value='timestamp']")
def test_timeseries_single_bucket(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#graph_type", state="attached")
page.fill("#start", "2024-01-01 00:00:00")
page.fill("#end", "2024-01-01 00:00:00")
select_value(page, "#graph_type", "timeseries")
page.click("text=Columns")
page.check("#column_groups input[value='value']")
page.click("text=View Settings")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
path = page.get_attribute("#chart path", "d")
assert path is not None and "NaN" not in path
(scubaduck) root@95eefccee427:/workspace/scubaduck#I'll create a timeseries test using the test_dataset_server_url and related settings. Let's explore nearby functions.
(scubaduck) root@95eefccee427:/workspace/scubaduck# grep -n "time_column" -n tes
ts/test_web_timeseries.py
(scubaduck) root@95eefccee427:/workspace/scubaduck#I’ll run a ruff format check before fixing the updateSelectedColumns function to include 'Count'.
(scubaduck) root@95eefccee427:/workspace/scubaduck# git status --short
(scubaduck) root@95eefccee427:/workspace/scubaduck#I'll open pyproject to review the dependencies I need for the tests.
name = "scubaduck"
version = "0.1.0"
description = "Scuba on DuckDB"
readme = "README.md"
requires-python = ">=3.12"
license = {text = "MIT"}
dependencies = [
"duckdb",
"flask",
"python-dateutil>=2.9.0.post0",
]
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"
[tool.pytest.ini_options]
addopts = "-n auto --tb=native"
[tool.pyright]
# Pyright configuration with strict settings
include = ["scubaduck", "tests"]
exclude = ["**/__pycache__", "dist"]
stubPath = "stubs"
venvPath = "."
venv = ".venv"
reportMissingImports = true
reportMissingTypeStubs = true
pythonVersion = "3.12"
pythonPlatform = "All"
typeCheckingMode = "strict"
reportUnknownMemberType = true
reportUnknownParameterType = true
reportUnknownVariableType = true
reportUnknownArgumentType = true
reportPrivateImportUsage = true
reportUntypedFunctionDecorator = true
reportFunctionMemberAccess = true
reportIncompatibleMethodOverride = true
[tool.ruff]
target-version = "py312"
[tool.uv]
dev-dependencies = [
"pytest>=8.3.5",
"ruff>=0.11.2",
"pytest-xdist>=3.6.1",
"playwright>=1.52.0",
"pyright>=1.1.350",
"pytest-playwright>=0.7.0",
]
(scubaduck) root@95eefccee427:/workspace/scubaduck#I’ll head back to lines around 440 in view_settings.js to apply the patch.
(scubaduck) root@95eefccee427:/workspace/scubaduck# sed -n '440,460p' scubaduck/
static/js/view_settings.js
const base = allColumns.filter(name => {
const cb = document.querySelector(`#column_groups input[value="${name}"]`);
if (!cb || !cb.checked) return false;
if (type === 'table' && isStringColumn(name)) return false;
return true;
});
if (type === 'table' || type === 'timeseries') {
selectedColumns = groupBy.chips.slice();
if (document.getElementById('show_hits').checked) selectedColumns.push('Hits
');
const agg = document.getElementById('aggregate').value.toLowerCase();
if (!(type === 'table' && agg === 'count')) {
base.forEach(c => {
if (!selectedColumns.includes(c)) selectedColumns.push(c);
});
derivedColumns.forEach(dc => {
if (dc.include && !selectedColumns.includes(dc.name)) selectedColumns.pu
sh(dc.name);
});
}
} else {
selectedColumns = base.slice();
derivedColumns.forEach(dc => {
(scubaduck) root@95eefccee427:/workspace/scubaduck#I'll add the condition for ensuring the 'Count' column is included in selectedColumns.
(scubaduck) root@95eefccee427:/workspace/scubaduck# apply_patch <<'PATCH'
> *** Begin Patch
> *** Update File: scubaduck/static/js/view_settings.js
> @@
> - if (!(type === 'table' && agg === 'count')) {
> - base.forEach(c => {
> - if (!selectedColumns.includes(c)) selectedColumns.push(c);
> - });
> - derivedColumns.forEach(dc => {
> - if (dc.include && !selectedColumns.includes(dc.name)) selectedColumns
.push(dc.name);
> - });
> - }
> + if (!(type === 'table' && agg === 'count')) {
> + base.forEach(c => {
> + if (!selectedColumns.includes(c)) selectedColumns.push(c);
> + });
> + derivedColumns.forEach(dc => {
> + if (dc.include && !selectedColumns.includes(dc.name)) selectedColumns
.push(dc.name);
> + });
> + }
> + if (type === 'timeseries' && agg === 'count' && !selectedColumns.includes
('Count')) {
> + selectedColumns.push('Count');
> + }
> } else {
> *** End Patch
> PATCH
Success. Updated the following files:
M scubaduck/static/js/view_settings.js
(scubaduck) root@95eefccee427:/workspace/scubaduck#I'll open tests/test_web_timeseries.py and look for a good spot to add the test.
(scubaduck) root@95eefccee427:/workspace/scubaduck# grep -n "test_timeseries" -n
tests/test_web_timeseries.py | head
8:def test_timeseries_default_query(page: Any, server_url: str) -> None:
25:def test_timeseries_single_bucket(page: Any, server_url: str) -> None:
41:def test_timeseries_fill_options(page: Any, server_url: str) -> None:
74:def test_timeseries_hover_highlight(page: Any, server_url: str) -> None:
101:def test_timeseries_crosshair(page: Any, server_url: str) -> None:
132:def test_timeseries_crosshair_freeze(page: Any, server_url: str) -> None:
180:def test_timeseries_auto_timezone(browser: Any, server_url: str) -> None:
199:def test_timeseries_multi_series(page: Any, server_url: str) -> None:
224:def test_timeseries_resize(page: Any, server_url: str) -> None:
262:def test_timeseries_no_overflow(page: Any, server_url: str) -> None:
(scubaduck) root@95eefccee427:/workspace/scubaduck#I'll scroll down further to check for additional tests past line 262.
page.goto(server_url)
page.wait_for_selector("#graph_type", state="attached")
select_value(page, "#graph_type", "timeseries")
page.click("text=Columns")
page.check("#column_groups input[value='value']")
page.click("text=View Settings")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
overflow = page.evaluate(
"var v=document.getElementById('view'); v.scrollWidth > v.clientWidth"
)
assert not overflow
def test_timeseries_axis_ticks(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#graph_type", state="attached")
select_value(page, "#graph_type", "timeseries")
page.click("text=Columns")
page.check("#column_groups input[value='value']")
page.click("text=View Settings")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
page.wait_for_selector("#chart text.tick-label", state="attached")
count = page.eval_on_selector_all("#chart text.tick-label", "els => els.leng
th")
assert count > 2
def test_timeseries_y_axis_labels(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#graph_type", state="attached")
select_value(page, "#graph_type", "timeseries")
page.click("text=Columns")
page.check("#column_groups input[value='value']")
page.click("text=View Settings")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
page.wait_for_selector("#chart text.y-tick-label", state="attached")
count = page.eval_on_selector_all("#chart text.y-tick-label", "els => els.le
ngth")
grid_count = page.eval_on_selector_all("#chart line.grid", "els => els.lengt
h")
assert count > 0 and count == grid_count
def test_timeseries_interval_offset(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#graph_type", state="attached")
select_value(page, "#graph_type", "timeseries")
page.click("text=Columns")
page.check("#column_groups input[value='value']")
page.click("text=View Settings")
page.fill("#start", "2024-01-01 00:00:00")
page.fill("#end", "2024-01-03 12:00:00")
select_value(page, "#granularity", "1 hour")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
page.wait_for_selector("#chart text.tick-label", state="attached")
labels = page.eval_on_selector_all(
"#chart text.tick-label", "els => els.map(e => e.textContent)"
)
assert labels
assert all(lbl != "00:00" for lbl in labels)
times = [lbl for lbl in labels if ":" in lbl]
assert times
for t in times:
h = int(t.split(":")[0])
assert h % 4 == 0
def test_timeseries_legend_values(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#graph_type", state="attached")
select_value(page, "#graph_type", "timeseries")
page.click("text=Columns")
page.check("#column_groups input[value='value']")(scubaduck) root@95eefccee427:/workspace/scubaduck# sed -n '480,640p' tests/test
_web_timeseries.py
(scubaduck) root@95eefccee427:/workspace/scubaduck#I’ll make sure to add the test near the end, but will verify after line 320.
meseries.py
def test_timeseries_group_links(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#graph_type", state="attached")
select_value(page, "#graph_type", "timeseries")
page.click("text=Columns")
page.check("#column_groups input[value='value']")
page.click("text=View Settings")
page.fill("#start", "2024-01-01 00:00:00")
page.fill("#end", "2024-01-02 03:00:00")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
assert page.text_content("#legend .drill-links h4") == "Group by"
page.click("#legend .drill-links a:text('user')")
page.wait_for_function("window.lastResults !== undefined")
chips = page.evaluate("groupBy.chips")
assert chips == ["user"]
assert page.text_content("#legend .drill-links h4") == "Drill up"
assert page.is_visible("#legend .drill-links a:text('Aggregate')")
(scubaduck) root@95eefccee427:/workspace/scubaduck#I'll look at tests/test_web_table.py to understand the style before adding my new test.
(scubaduck) root@95eefccee427:/workspace/scubaduck# grep -n "test_dataset_server
_url" -n tests/test_web_table.py
16:def test_dataset_server_url() -> Iterator[str]:
333:def test_table_count_no_columns(page: Any, test_dataset_server_url: str) ->
None:
334: page.goto(test_dataset_server_url)
(scubaduck) root@95eefccee427:/workspace/scubaduck#I'll start by opening the beginning of test_web_table.py to review how the fixture is used.
_web_table.py
page,
server_url,
start="2024-01-01 00:00:00",
end="2024-01-02 00:00:00",
order_by="timestamp",
limit=10,
)
sql = data["sql"]
displayed = page.text_content("#sql_query")
assert displayed is not None
assert displayed.strip() == sql
def test_table_count_no_columns(page: Any, test_dataset_server_url: str) -> None
:
page.goto(test_dataset_server_url)
page.wait_for_selector("#order_by option", state="attached")
select_value(page, "#graph_type", "table")
page.click("text=Columns")
page.click("#columns_all")
page.click("text=View Settings")
page.evaluate("groupBy.chips = ['id']; groupBy.renderChips();")
select_value(page, "#aggregate", "Count")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
headers = page.locator("#results th").all_inner_texts()
assert headers == ["id", "Hits"]
col_count = page.locator("#results th").count()
row_count = page.locator("#results tr").count()
assert col_count == 2
assert row_count == 3
overflow = page.evaluate(
"var v=document.getElementById('view'); v.scrollWidth > v.clientWidth"
)
assert not overflow
(scubaduck) root@95eefccee427:/workspace/scubaduck#I'll design a test for the timeseries count bug, ensuring the chart is visible and valid.
I'll confirm the chart path count is 1 and verify time_column and time_unit settings.
(scubaduck) root@95eefccee427:/workspace/scubaduck# grep -n "time_column" -n scu
baduck/static/index.html
204: <select id="time_column"></select>
355:<script src="/js/time_column.js"></script>(scubaduck) root@95eefccee427:/workspace/scubaduck# sed -n '180,228p' scubaduck/
static/index.html
text-anchor: end;
}
#chart text.y-tick-label {
font-size: 10px;
user-select: none;
}
#chart line.grid {
stroke: #ccc;
}
/* Column resizer removed */
</style>
</head>
<body>
<div id="header">sample.csv - <select id="table"></select> <select id="graph_t
ype"><option value="samples">Samples</option><option value="table">Table</option
><option value="timeseries">Time Series</option></select></div>
<div id="content">
<div id="sidebar">
<div id="tabs">
<button class="tab active" data-tab="settings">View Settings</button>
<button id="columns_tab" class="tab" data-tab="columns">Columns</button>
<button id="dive" onclick="dive()">Dive</button>
</div>
<div id="settings" class="tab-content active">
<div class="field">
<label>Time Column</label>
<select id="time_column"></select>
<select id="time_unit" style="margin-left:4px">
<option value="s">s</option>
<option value="ms">ms</option>
<option value="us">us</option>
<option value="ns">ns</option>
</select>
</div>
<div class="field">
<label>Start<span class="help" title="Sets the start/end of the time r
ange to query. Can be any kind of datetime string. For example: 'April 23, 2014'
or 'yesterday'.">[?]</span></label>
<div class="rel-box">
<input id="start" type="text" />
<button type="button" class="rel-btn" data-target="start-select">	
660;</button>
<div id="start-select" class="rel-dropdown" data-input="start">
<div data-value="-1 hour">-1 hour</div>
<div data-value="-3 hours">-3 hours</div>
<div data-value="-12 hours">-12 hours</div>
<div data-value="-1 day">-1 day</div>
<div data-value="-3 days">-3 days</div>
<div data-value="-1 week">-1 week</div>
<div data-value="-1 fortnight">-1 fortnight</div>
<div data-value="-30 days">-30 days</div>
<div data-value="-90 days">-90 days</div>
</div>
</div>
(scubaduck) root@95eefccee427:/workspace/scubaduck#I'll check index.html/js to find the ID for the "select all" columns option.
(scubaduck) root@95eefccee427:/workspace/scubaduck# grep -n "columns_all" -r scu
baduck/static | head
scubaduck/static/js/view_settings.js:335: document.getElementById('columns_al
l').addEventListener('click', e => {
scubaduck/static/index.html:337: <a id="columns_all" href="#">All</a>(scubaduck) root@95eefccee427:/workspace/scubaduck# sed -n '324,352p' scubaduck/
static/index.html
<div id="show_hits_field" class="field" style="display:none;">
<label>Show Hits</label>
<input id="show_hits" type="checkbox" checked>
</div>
<div id="filters">
<h4>Filters<span class="help" title="You can create as many filters as
you want. You can either write a filter using a UI or manual SQL. In the UI, fi
lter consists of a column name, a relation (e.g., =, !=, <, >) and then a text f
ield. The text field is a token input. It accepts multiple tokens for = relation
, in which case we match using an OR for all options.">[?]</span></h4>
<div id="filter_list"></div>
<button id="add_filter" type="button" onclick="addFilter()">Add Filter
</button>
</div>
<div id="query_info" style="margin-top:10px;"></div>
</div>
<div id="columns" class="tab-content">
<div id="column_actions">
<a id="columns_all" href="#">All</a>
<a id="columns_none" href="#">None</a>
</div>
<div id="column_groups"></div>
<div id="derived_columns">
<h4>Derived Columns</h4>
<div id="derived_list"></div>
<button id="add_derived" type="button" onclick="addDerived()">Add Deri
ved</button>
</div>
</div>
</div>
<div id="sidebar-resizer"></div>
<div id="view">
<table id="results"></table>
</div>
</div>
(scubaduck) root@95eefccee427:/workspace/scubaduck#I'll add the test at the bottom of test_web_timeseries.py after test_timeseries_group_links.
(scubaduck) root@95eefccee427:/workspace/scubaduck# grep -n "columns_none" -n sc
ubaduck/static/js/view_settings.js
340: document.getElementById('columns_none').addEventListener('click', e => {(scubaduck) root@95eefccee427:/workspace/scubaduck# sed -n '332,352p' scubaduck/
static/js/view_settings.js
div.appendChild(ul);
groupsEl.appendChild(div);
});
document.getElementById('columns_all').addEventListener('click', e => {
e.preventDefault();
groupsEl.querySelectorAll('input').forEach(cb => (cb.checked = true));
updateSelectedColumns();
});
document.getElementById('columns_none').addEventListener('click', e => {
e.preventDefault();
groupsEl.querySelectorAll('input').forEach(cb => (cb.checked = false));
updateSelectedColumns();
});
updateSelectedColumns();
columnValues.samples = allColumns.slice();
columnValues.table = [];
columnValues.timeseries = [];
groupBy = document.getElementById('group_by').closest('.field');
initChipInput(groupBy, typed =>
allColumns.filter(c => c.toLowerCase().includes(typed.toLowerCase()))
);
(scubaduck) root@95eefccee427:/workspace/scubaduck##columns_none unchecks all checkboxes. I'll search for the IDs for time_unit and time_column.
(scubaduck) root@95eefccee427:/workspace/scubaduck# grep -n "time_unit" -n scuba
duck/static/index.html
205: <select id="time_unit" style="margin-left:4px">
(scubaduck) root@95eefccee427:/workspace/scubaduck#I'll draft the test, ensuring I cover the checkbox's default behavior and the show's hits logic.
I’ll ensure that 'Hits' and 'Count' are correctly selected, even with show_hits.
I'll add a check for valid path values before asserting the path count.
> *** Begin Patch
> *** Update File: tests/test_web_timeseries.py
> @@
> def test_timeseries_group_links(page: Any, server_url: str) -> None:
> @@
> assert page.is_visible("#legend .drill-links a:text('Aggregate')")
> +
> +
> +def test_timeseries_count_no_columns_numeric_time(
> + page: Any, test_dataset_server_url: str
> +) -> None:
> + page.goto(test_dataset_server_url)
> + page.wait_for_selector("#graph_type", state="attached")
> + select_value(page, "#graph_type", "timeseries")
> + page.click("text=Columns")
> + page.click("#columns_none")
> + page.click("text=View Settings")
> + select_value(page, "#time_column", "ts")
> + select_value(page, "#time_unit", "s")
> + select_value(page, "#aggregate", "Count")
> + page.evaluate("window.lastResults = undefined")
> + page.click("text=Dive")
> + page.wait_for_function("window.lastResults !== undefined")
> + page.wait_for_selector("#chart path", state="attached")
> + series_count = page.eval_on_selector_all("#chart path", "els => els.lengt
h")
> + assert series_count == 1
> *** End Patch
> PATCH
Success. Updated the following files:
M tests/test_web_timeseries.py
(scubaduck) root@95eefccee427:/workspace/scubaduck#I'll check the ruff formatting after inserting the patch near test_timeseries_group_links.
(scubaduck) root@95eefccee427:/workspace/scubaduck# ruff format scubaduck/static
/js/view_settings.js tests/test_web_timeseries.py >/tmp/ruff_format.log && tail
-n 20 /tmp/ruff_format.log
error: Failed to parse scubaduck/static/js/view_settings.js:1:1: Expected a stat
ement
(scubaduck) root@95eefccee427:/workspace/scubaduck#Ruff only works for Python, so I'll run it on the Python file, test_web_timeseries.py.
(scubaduck) root@95eefccee427:/workspace/scubaduck# ruff format tests/test_web_t
imeseries.py >/tmp/ruff_format.log && tail -n 20 /tmp/ruff_format.log
1 file left unchanged(scubaduck) root@95eefccee427:/workspace/scubaduck# ruff check >/tmp/ruff_check.
log && tail -n 20 /tmp/ruff_check.log(scubaduck) root@95eefccee427:/workspace/scubaduck# pyright >/tmp/pyright.log &&
tail -n 20 /tmp/pyright.log(scubaduck) root@95eefccee427:/workspace/scubaduck# cat /tmp/pyright.log
/workspace/scubaduck/tests/test_server_errors.py
/workspace/scubaduck/tests/test_server_errors.py:65:5 - error: Variable "data"
is not accessed (reportUnusedVariable)
1 error, 0 warnings, 0 informations
(scubaduck) root@95eefccee427:/workspace/scubaduck#Pyright says there's an unused variable on line 65, but we didn't modify this file. Let's check.
(scubaduck) root@95eefccee427:/workspace/scubaduck# sed -n '60,70p' tests/test_s
erver_errors.py
"show_hits": True,
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
def test_samples_view_rejects_group_by() -> None:
app = server.app
(scubaduck) root@95eefccee427:/workspace/scubaduck#The output shows lines 60-70 but doesn't display the data variable—let me check again.
(scubaduck) root@95eefccee427:/workspace/scubaduck# sed -n '54,70p' tests/test_s
erver_errors.py
"graph_type": "table",
"order_by": "timestamp",
"limit": 100,
"columns": ["user", "Hits", "value"],
"group_by": ["user"],
"aggregate": "Count",
"show_hits": True,
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
def test_samples_view_rejects_group_by() -> None:
app = server.app
(scubaduck) root@95eefccee427:/workspace/scubaduck#Looks like data = rv.get_json() isn't used—maybe an intentional omission or missed pyright run. I'll check the commit history for context.
ors.py | sed -n '55,70p'
55 "order_by": "timestamp",
56 "limit": 100,
57 "columns": ["user", "Hits", "value"],
58 "group_by": ["user"],
59 "aggregate": "Count",
60 "show_hits": True,
61 }
62 rv = client.post(
63 "/api/query", data=json.dumps(payload), content_type="applicatio
n/json"
64 )
65 data = rv.get_json()
66 assert rv.status_code == 200
67
68
69 def test_samples_view_rejects_group_by() -> None:
70 app = server.app
(scubaduck) root@95eefccee427:/workspace/scubaduck#I’ll check the previous commit to verify if any changes affected pyright errors.
commit 2ac2fc5088f297ae0f0b00b319b420d450c710a7 (HEAD -> work, main)
Author: Edward Z. Yang <ezyang@mit.edu>
Date: Thu May 22 22:17:15 2025 -0400
fix(ui): reset view settings on table change (#144)
scubaduck/static/js/view_settings.js | 44 +++++++++++++++++++++++++++++++-----
tests/test_multi_table_web.py | 26 +++++++++++++++++++++
2 files changed, 64 insertions(+), 6 deletions(-)
(scubaduck) root@95eefccee427:/workspace/scubaduck#I see pyright used to run on HEAD and maybe suppressed unused variable errors. Let's check.
(scubaduck) root@95eefccee427:/workspace/scubaduck# git blame -L 60,67 tests/tes
t_server_errors.py
ee84b236 (Edward Z. Yang 2025-05-22 21:32:05 -0400 60) "show_hits": True
,
ee84b236 (Edward Z. Yang 2025-05-22 21:32:05 -0400 61) }
ee84b236 (Edward Z. Yang 2025-05-22 21:32:05 -0400 62) rv = client.post(
ee84b236 (Edward Z. Yang 2025-05-22 21:32:05 -0400 63) "/api/query", dat
a=json.dumps(payload), content_type="application/json"
ee84b236 (Edward Z. Yang 2025-05-22 21:32:05 -0400 64) )
ee84b236 (Edward Z. Yang 2025-05-22 21:32:05 -0400 65) data = rv.get_json()
9b4de028 (Edward Z. Yang 2025-05-22 22:16:07 -0400 66) assert rv.status_code
== 200
ee84b236 (Edward Z. Yang 2025-05-22 21:32:05 -0400 67)
(scubaduck) root@95eefccee427:/workspace/scubaduck#It looks like the error might be ignored because pyright isn't running on tests. I'll check the configuration.
(scubaduck) root@95eefccee427:/workspace/scubaduck# git show 2ac2fc5088f297ae0f0
b00b319b420d450c710a7~1:pyproject.toml | grep reportUnusedVariable
(scubaduck) root@95eefccee427:/workspace/scubaduck#I'll check the full config from the commit to confirm there are no changes.
(scubaduck) root@95eefccee427:/workspace/scubaduck# git show 2ac2fc5088f297ae0f0
b00b319b420d450c710a7~1:pyproject.toml | head
[project]
name = "scubaduck"
version = "0.1.0"
description = "Scuba on DuckDB"
readme = "README.md"
requires-python = ">=3.12"
license = {text = "MIT"}
dependencies = [
"duckdb",
"flask",
(scubaduck) root@95eefccee427:/workspace/scubaduck#The variable might have caused the Pyright error. I'll check the Pyright log for more details.
(scubaduck) root@95eefccee427:/workspace/scubaduck# grep -n "error" -n /tmp/pyri
ght.log
1:/workspace/scubaduck/tests/test_server_errors.py
2: /workspace/scubaduck/tests/test_server_errors.py:65:5 - error: Variable "dat
a" is not accessed (reportUnusedVariable)
3:1 error, 0 warnings, 0 informations
(scubaduck) root@95eefccee427:/workspace/scubaduck#I'll patch the issue by renaming the variable to _data or simply removing it.
(scubaduck) root@95eefccee427:/workspace/scubaduck# sed -n '58,66p' tests/test_s
erver_errors.py
"group_by": ["user"],
"aggregate": "Count",
"show_hits": True,
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200(scubaduck) root@95eefccee427:/workspace/scubaduck# apply_patch <<'PATCH'
> *** Begin Patch
> *** Update File: tests/test_server_errors.py
> @@
> - data = rv.get_json()
> - assert rv.status_code == 200
> + _data = rv.get_json()
> + assert rv.status_code == 200
> *** End Patch
> PATCH
Success. Updated the following files:
M tests/test_server_errors.py
(scubaduck) root@95eefccee427:/workspace/scubaduck#I'll run ruff format on the file again to ensure everything’s clean.
(scubaduck) root@95eefccee427:/workspace/scubaduck# ruff format tests/test_serve
r_errors.py tests/test_web_timeseries.py >/tmp/ruff_format.log && tail -n 20 /tm
p/ruff_format.log
2 files left unchanged
(scubaduck) root@95eefccee427:/workspace/scubaduck#I'll go ahead and execute the ruff check again to ensure everything's in order.
(scubaduck) root@95eefccee427:/workspace/scubaduck# ruff check >/tmp/ruff_check.
log && tail -n 20 /tmp/ruff_check.log
All checks passed!
(scubaduck) root@95eefccee427:/workspace/scubaduck#I need to run pyright once more. Let's give it another shot!
(scubaduck) root@95eefccee427:/workspace/scubaduck# pyright >/tmp/pyright.log &&
tail -n 20 /tmp/pyright.log0 errors, 0 warnings, 0 informations
(scubaduck) root@95eefccee427:/workspace/scubaduck#I'll double-check the log to confirm the details.
(scubaduck) root@95eefccee427:/workspace/scubaduck# cat /tmp/pyright.log
0 errors, 0 warnings, 0 informations(scubaduck) root@95eefccee427:/workspace/scubaduck# pytest >/tmp/pytest.log && t
ail -n 20 /tmp/pytest.logIt looks like the tests are running, but the output is hidden. I'll check the log later.
Looks like tests are taking a while. I'll check the /tmp/pytest.log for updates.
(scubaduck) root@95eefccee427:/workspace/scubaduck# tail -n 20 /tmp/pytest.log
INFO werkzeug:_internal.py:97 127.0.0.1 - - [23/May/2025 02:31:07] "GET /js/
view_settings.js HTTP/1.1" 200 -
INFO werkzeug:_internal.py:97 127.0.0.1 - - [23/May/2025 02:31:07] "GET /api
/tables HTTP/1.1" 200 -
INFO werkzeug:_internal.py:97 127.0.0.1 - - [23/May/2025 02:31:07] "GET /api
/columns?table=events HTTP/1.1" 200 -
INFO werkzeug:_internal.py:97 127.0.0.1 - - [23/May/2025 02:31:07] "POST /ap
i/query HTTP/1.1" 200 -
INFO werkzeug:_internal.py:97 127.0.0.1 - - [23/May/2025 02:31:07] "POST /ap
i/query HTTP/1.1" 400 -
=========================== short test summary info ============================
FAILED tests/test_web_timeseries.py::test_timeseries_default_query[chromium]
FAILED tests/test_web_timeseries.py::test_timeseries_crosshair[chromium] - pl...
FAILED tests/test_web_timeseries.py::test_timeseries_single_bucket[chromium]
FAILED tests/test_web_timeseries.py::test_timeseries_resize[chromium] - playw...
FAILED tests/test_web_timeseries.py::test_timeseries_interval_offset[chromium]
FAILED tests/test_web_timeseries.py::test_timeseries_crosshair_freeze[chromium]
FAILED tests/test_web_timeseries.py::test_timeseries_fill_options[chromium]
FAILED tests/test_web_timeseries.py::test_timeseries_axis_ticks[chromium] - p...
FAILED tests/test_web_timeseries.py::test_timeseries_group_links[chromium] - ...
FAILED tests/test_web_timeseries.py::test_timeseries_auto_timezone[chromium]
FAILED tests/test_web_timeseries.py::test_timeseries_hover_highlight[chromium]
FAILED tests/test_web_timeseries.py::test_timeseries_y_axis_labels[chromium]
ERROR tests/test_web_timeseries.py::test_timeseries_count_no_columns_numeric_tim
e[chromium]
============= 12 failed, 108 passed, 1 error in 104.00s (0:01:44) ==============
(scubaduck) root@95eefccee427:/workspace/scubaduck#Test failures might be due to our changes to the timeseries aggregator. I'll review errors.
(scubaduck) root@95eefccee427:/workspace/scubaduck# grep -n "FAILED" -n /tmp/pyt
est.log
1201:FAILED tests/test_web_timeseries.py::test_timeseries_default_query[chromium
]
1202:FAILED tests/test_web_timeseries.py::test_timeseries_crosshair[chromium] -
pl...
1203:FAILED tests/test_web_timeseries.py::test_timeseries_single_bucket[chromium
]
1204:FAILED tests/test_web_timeseries.py::test_timeseries_resize[chromium] - pla
yw...
1205:FAILED tests/test_web_timeseries.py::test_timeseries_interval_offset[chromi
um]
1206:FAILED tests/test_web_timeseries.py::test_timeseries_crosshair_freeze[chrom
ium]
1207:FAILED tests/test_web_timeseries.py::test_timeseries_fill_options[chromium]
1208:FAILED tests/test_web_timeseries.py::test_timeseries_axis_ticks[chromium] -
p...
1209:FAILED tests/test_web_timeseries.py::test_timeseries_group_links[chromium]
- ...
1210:FAILED tests/test_web_timeseries.py::test_timeseries_auto_timezone[chromium
]
1211:FAILED tests/test_web_timeseries.py::test_timeseries_hover_highlight[chromi
um]
1212:FAILED tests/test_web_timeseries.py::test_timeseries_y_axis_labels[chromium
]
(scubaduck) root@95eefccee427:/workspace/scubaduck#I’ll check the earlier lines to understand where the failure occurred.
(scubaduck) root@95eefccee427:/workspace/scubaduck# grep -n "Time Series" -n /tm
p/pytest.log | head
(scubaduck) root@95eefccee427:/workspace/scubaduck#I think there might be a syntax issue causing a page load failure—I'll check the patch.
(scubaduck) root@95eefccee427:/workspace/scubaduck# sed -n '448,462p' scubaduck/
static/js/view_settings.js
if (document.getElementById('show_hits').checked) selectedColumns.push('Hits
');
const agg = document.getElementById('aggregate').value.toLowerCase();
if (!(type === 'table' && agg === 'count')) {
base.forEach(c => {
if (!selectedColumns.includes(c)) selectedColumns.push(c);
});
derivedColumns.forEach(dc => {
if (dc.include && !selectedColumns.includes(dc.name)) selectedColumns.pu
sh(dc.name);
});
}
if (type === 'timeseries' && agg === 'count' && !selectedColumns.includes('C
ount')) {
selectedColumns.push('Count');
}
} else {
selectedColumns = base.slice();
(scubaduck) root@95eefccee427:/workspace/scubaduck#I suspect the failure happens due to how 'Count' is pushed in the wrong place.
(scubaduck) root@95eefccee427:/workspace/scubaduck# grep -n "FAILURES" -n /tmp/p
ytest.log
21:=================================== FAILURES ================================
===(scubaduck) root@95eefccee427:/workspace/scubaduck# sed -n '21,120p' /tmp/pytest
.log
=================================== FAILURES ===================================
___________________ test_timeseries_default_query[chromium] ____________________
[gw1] linux -- Python 3.12.3 /workspace/scubaduck/.venv/bin/python
Traceback (most recent call last):
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/runner.p
y", line 341, in from_call
result: TResult | None = func()
^^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/runner.p
y", line 242, in <lambda>
lambda: runtest_hook(item=item, **kwds), when=when, reraise=reraise
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_hooks.py
", line 512, in __call__
return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult
)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_manager.
py", line 120, in _hookexec
return self._inner_hookexec(hook_name, methods, kwargs, firstresult)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_callers.
py", line 167, in _multicall
raise exception
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_callers.
py", line 139, in _multicall
teardown.throw(exception)
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/threadex
ception.py", line 92, in pytest_runtest_call
yield from thread_exception_runtest_hook()
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/threadex
ception.py", line 68, in thread_exception_runtest_hook
yield
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_callers.
py", line 139, in _multicall
teardown.throw(exception)
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/unraisab
leexception.py", line 95, in pytest_runtest_call
yield from unraisable_exception_runtest_hook()
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/unraisab
leexception.py", line 70, in unraisable_exception_runtest_hook
yield
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_callers.
py", line 139, in _multicall
teardown.throw(exception)
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/logging.
py", line 846, in pytest_runtest_call
yield from self._runtest_for(item, "call")
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/logging.
py", line 829, in _runtest_for
yield
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_callers.
py", line 139, in _multicall
teardown.throw(exception)
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/capture.
py", line 898, in pytest_runtest_call
return (yield)
^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_callers.
py", line 139, in _multicall
teardown.throw(exception)
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/skipping
.py", line 257, in pytest_runtest_call
return (yield)
^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_callers.
py", line 121, in _multicall
res = hook_impl.function(*args)
^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/runner.p
y", line 174, in pytest_runtest_call
item.runtest()
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/python.p
y", line 1627, in runtest
self.ihook.pytest_pyfunc_call(pyfuncitem=self)
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_hooks.py
", line 512, in __call__
return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult
)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_manager.
py", line 120, in _hookexec
return self._inner_hookexec(hook_name, methods, kwargs, firstresult)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_callers.
py", line 167, in _multicall
raise exception
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_callers.
py", line 121, in _multicall
res = hook_impl.function(*args)
^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/python.p
y", line 159, in pytest_pyfunc_call
result = testfunction(**testargs)
^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/scubaduck/tests/test_web_timeseries.py", line 19, in test_tim
eseries_default_query
assert "error" not in data
AssertionError: assert 'error' not in {'error': 'Unknown column: Count'}
------------------------------ Captured log call -------------------------------
INFO werkzeug:_internal.py:97 127.0.0.1 - - [23/May/2025 02:30:04] "GET / HT
TP/1.1" 200 -
INFO werkzeug:_internal.py:97 127.0.0.1 - - [23/May/2025 02:30:04] "GET /js/
chip_input.js HTTP/1.1" 200 -
INFO werkzeug:_internal.py:97 127.0.0.1 - - [23/May/2025 02:30:04] "GET /js/
timeseries_chart.js HTTP/1.1" 200 -
INFO werkzeug:_internal.py:97 127.0.0.1 - - [23/May/2025 02:30:04] "GET /js/
time_column.js HTTP/1.1" 200 -
INFO werkzeug:_internal.py:97 127.0.0.1 - - [23/May/2025 02:30:04] "GET /js/
sidebar.js HTTP/1.1" 200 -
INFO werkzeug:_internal.py:97 127.0.0.1 - - [23/May/2025 02:30:04] "GET /js/
table_view.js HTTP/1.1" 200 -
INFO werkzeug:_internal.py:97 127.0.0.1 - - [23/May/2025 02:30:04] "GET /js/
view_settings.js HTTP/1.1" 200 -
INFO werkzeug:_internal.py:97 127.0.0.1 - - [23/May/2025 02:30:04] "GET /api
/tables HTTP/1.1" 200 -
INFO werkzeug:_internal.py:97 127.0.0.1 - - [23/May/2025 02:30:04] "GET /api
/columns?table=events HTTP/1.1" 200 -
INFO werkzeug:_internal.py:97 127.0.0.1 - - [23/May/2025 02:30:04] "POST /ap
i/query HTTP/1.1" 200 -
INFO werkzeug:_internal.py:97 127.0.0.1 - - [23/May/2025 02:30:04] "POST /ap
i/query HTTP/1.1" 400 -
_____________________ test_timeseries_crosshair[chromium] ______________________
[gw0] linux -- Python 3.12.3 /workspace/scubaduck/.venv/bin/python
Traceback (most recent call last):
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/runner.p
y", line 341, in from_call
result: TResult | None = func()
^^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/runner.p
y", line 242, in <lambda>
lambda: runtest_hook(item=item, **kwds), when=when, reraise=reraise
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_hooks.py
", line 512, in __call__
return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult
)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_manager.
py", line 120, in _hookexec
return self._inner_hookexec(hook_name, methods, kwargs, firstresult)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_callers.
py", line 167, in _multicall
raise exception
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_callers.
py", line 139, in _multicall(scubaduck) root@95eefccee427:/workspace/scubaduck# sed -n '120,180p' /tmp/pytes
t.log
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_callers.
py", line 139, in _multicall
teardown.throw(exception)
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/threadex
ception.py", line 92, in pytest_runtest_call
yield from thread_exception_runtest_hook()
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/threadex
ception.py", line 68, in thread_exception_runtest_hook
yield
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_callers.
py", line 139, in _multicall
teardown.throw(exception)
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/unraisab
leexception.py", line 95, in pytest_runtest_call
yield from unraisable_exception_runtest_hook()
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/unraisab
leexception.py", line 70, in unraisable_exception_runtest_hook
yield
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_callers.
py", line 139, in _multicall
teardown.throw(exception)
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/logging.
py", line 846, in pytest_runtest_call
yield from self._runtest_for(item, "call")
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/logging.
py", line 829, in _runtest_for
yield
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_callers.
py", line 139, in _multicall
teardown.throw(exception)
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/capture.
py", line 898, in pytest_runtest_call
return (yield)
^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_callers.
py", line 139, in _multicall
teardown.throw(exception)
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/skipping
.py", line 257, in pytest_runtest_call
return (yield)
^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_callers.
py", line 121, in _multicall
res = hook_impl.function(*args)
^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/runner.p
y", line 174, in pytest_runtest_call
item.runtest()
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/python.p
y", line 1627, in runtest
self.ihook.pytest_pyfunc_call(pyfuncitem=self)
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_hooks.py
", line 512, in __call__
return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult
)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_manager.
py", line 120, in _hookexec
return self._inner_hookexec(hook_name, methods, kwargs, firstresult)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_callers.
py", line 167, in _multicall
raise exception
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_callers.
py", line 121, in _multicall
res = hook_impl.function(*args)
^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/python.p
y", line 159, in pytest_pyfunc_call
result = testfunction(**testargs)
^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/scubaduck/tests/test_web_timeseries.py", line 111, in test_ti
meseries_crosshair
page.wait_for_selector("#chart path", state="attached")
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/playwright/sync_
api/_generated.py", line 8187, in wait_for_selector
self._sync(
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/playwright/_impl
/_sync_base.py", line 115, in _sync
return task.result()
^^^^^^^^^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/playwright/_impl
/_page.py", line 425, in wait_for_selector
return await self._main_frame.wait_for_selector(**locals_to_params(locals())
)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/playwright/_impl
/_frame.py", line 323, in wait_for_selector
await self._channel.send("waitForSelector", locals_to_params(locals()))
(scubaduck) root@95eefccee427:/workspace/scubaduck#The error likely happens due to the default "Count" in the aggregate dropdown when it should be "Avg". I'll check index.html to confirm the default settings.
(scubaduck) root@95eefccee427:/workspace/scubaduck# grep -n "aggregate" -n scuba
duck/static/index.html | head
303: <div id="aggregate_field" class="field" style="display:none;">
305: <select id="aggregate">(scubaduck) root@95eefccee427:/workspace/scubaduck# sed -n '300,316p' scubaduck/
static/index.html
<div class="chip-dropdown"></div>
</div>
</div>
<div id="aggregate_field" class="field" style="display:none;">
<label>Aggregate</label>
<select id="aggregate">
<option>Count</option>
<option>Avg</option>
<option>Sum</option>
<option>Min</option>
<option>Max</option>
<option>Count Distinct</option>
<option>p5</option>
<option>p25</option>
<option>p50</option>
<option>p70</option>
<option>p75</option>(scubaduck) root@95eefccee427:/workspace/scubaduck# sed -n '316,332p' scubaduck/
static/index.html
<option>p75</option>
<option>p90</option>
<option>p95</option>
<option>p99</option>
<option>p99.9</option>
<option>p99.99</option>
</select>
</div>
<div id="show_hits_field" class="field" style="display:none;">
<label>Show Hits</label>
<input id="show_hits" type="checkbox" checked>
</div>
<div id="filters">
<h4>Filters<span class="help" title="You can create as many filters as
you want. You can either write a filter using a UI or manual SQL. In the UI, fi
lter consists of a column name, a relation (e.g., =, !=, <, >) and then a text f
ield. The text field is a token input. It accepts multiple tokens for = relation
, in which case we match using an OR for all options.">[?]</span></h4>
<div id="filter_list"></div>
<button id="add_filter" type="button" onclick="addFilter()">Add Filter
</button>
</div>
(scubaduck) root@95eefccee427:/workspace/scubaduck#The test expects aggregator not to be 'Count,' but that changed with my patch.
(scubaduck) root@95eefccee427:/workspace/scubaduck# grep -n "columns: selectedCo
lumns.filter" -n scubaduck/static/js/view_settings.js
705: columns: selectedColumns.filter(c =>(scubaduck) root@95eefccee427:/workspace/scubaduck# sed -n '704,712p' scubaduck/
static/js/view_settings.js
limit: parseInt(document.getElementById('limit').value, 10),
columns: selectedColumns.filter(c =>
c !== 'Hits' && !derivedColumns.some(dc => dc.name === c)
),
samples_columns: columnValues.samples.slice(),
table_columns: columnValues.table.slice(),
timeseries_columns: columnValues.timeseries.slice(),
graph_type: graphTypeSel.value,
filters: Array.from(document.querySelectorAll('#filters .filter')).map(f =>
{
(scubaduck) root@95eefccee427:/workspace/scubaduck#The "Count" column is mistakenly added to the query. I'll filter it out in collectParams.
I could adjust showTimeSeries to treat Hits as value if aggregator is count.
(scubaduck) root@95eefccee427:/workspace/scubaduck# grep -n "Hits" -n scubaduck/
static/js/timeseries_chart.js
32: const hasHits = document.getElementById('show_hits').checked ? 1 : 0;
37: const startIdx = 1 + groups.length + hasHits;
38: const valueCols = selectedColumns.slice(groups.length + hasHits);
(scubaduck) root@95eefccee427:/workspace/scubaduck#I'll modify the code to handle empty valueCols, setting 'Count' when needed, and update legend.
I'll update showTimeSeries to match the logic for populating valueCols and names.
(scubaduck) root@95eefccee427:/workspace/scubaduck# sed -n '28,46p' scubaduck/st
atic/js/timeseries_chart.js
const crosshairDots = document.createElementNS('http://www.w3.org/2000/svg', '
g');
crosshairDots.id = 'crosshair_dots';
crosshairDots.style.display = 'none';
const groups = groupBy.chips || [];
const hasHits = document.getElementById('show_hits').checked ? 1 : 0;
const fill = document.getElementById('fill').value;
const bucketMs = (data.bucket_size || 3600) * 1000;
const start = data.start ? parseTs(data.start) : null;
const end = data.end ? parseTs(data.end) : null;
const startIdx = 1 + groups.length + hasHits;
const valueCols = selectedColumns.slice(groups.length + hasHits);
const series = {};
data.rows.forEach(r => {
const ts = parseTs(r[0]);
const groupKey = groups.map((_, i) => r[1 + i]).join(':') || 'all';
valueCols.forEach((name, i) => {
const val = Number(r[startIdx + i]);
const key = groupKey === 'all' ? name : groupKey + ':' + name;
if (!series[key]) series[key] = {};
(scubaduck) root@95eefccee427:/workspace/scubaduck#I'll go ahead and apply the patch.
(scubaduck) root@95eefccee427:/workspace/scubaduck# apply_patch <<'PATCH'
> *** Begin Patch
> *** Update File: scubaduck/static/js/timeseries_chart.js
> @@
> - const valueCols = selectedColumns.slice(groups.length + hasHits);
> + let valueCols = selectedColumns.slice(groups.length + hasHits);
> + if (
> + valueCols.length === 0 &&
> + document.getElementById('aggregate').value.toLowerCase() === 'count'
> + ) {
> + valueCols = ['Count'];
> + }
> *** End Patch
> PATCH
Success. Updated the following files:
M scubaduck/static/js/timeseries_chart.js
(scubaduck) root@95eefccee427:/workspace/scubaduck#I realized the 'Count' addition isn't necessary anymore, so I'll revert it in view_settings.js.
(scubaduck) root@95eefccee427:/workspace/scubaduck# git checkout -- scubaduck/st
atic/js/view_settings.js
(scubaduck) root@95eefccee427:/workspace/scubaduck#I’ll confirm if the patch is fully reverted along with any additional changes.
(scubaduck) root@95eefccee427:/workspace/scubaduck# grep -n "Count'" -n scubaduc
k/static/js/view_settings.js
848: document.getElementById('aggregate').value = 'Count';
(scubaduck) root@95eefccee427:/workspace/scubaduck#I'll open the snippet now to double-check the patch was successfully reverted.
(scubaduck) root@95eefccee427:/workspace/scubaduck# sed -n '448,462p' scubaduck/
static/js/view_settings.js
if (document.getElementById('show_hits').checked) selectedColumns.push('Hits
');
const agg = document.getElementById('aggregate').value.toLowerCase();
if (!(type === 'table' && agg === 'count')) {
base.forEach(c => {
if (!selectedColumns.includes(c)) selectedColumns.push(c);
});
derivedColumns.forEach(dc => {
if (dc.include && !selectedColumns.includes(dc.name)) selectedColumns.pu
sh(dc.name);
});
}
} else {
selectedColumns = base.slice();
derivedColumns.forEach(dc => {
if (dc.include) selectedColumns.push(dc.name);
});
(scubaduck) root@95eefccee427:/workspace/scubaduck#I’ve reverted the changes. Let me run ruff format on the updated files now!
(scubaduck) root@95eefccee427:/workspace/scubaduck# ruff format scubaduck/static
/js/timeseries_chart.js tests/test_web_timeseries.py tests/test_server_errors.py
>/tmp/ruff_format.log && tail -n 20 /tmp/ruff_format.log
error: Failed to parse scubaduck/static/js/timeseries_chart.js:1:5: Simple state
ments must be separated by newlines or semicolons
(scubaduck) root@95eefccee427:/workspace/scubaduck#Since ruff can't parse JS, I'll restrict its use to Python files.
(scubaduck) root@95eefccee427:/workspace/scubaduck# ruff format tests/test_web_t
imeseries.py tests/test_server_errors.py >/tmp/ruff_format.log && tail -n 20 /tm
p/ruff_format.log
2 files left unchanged
(scubaduck) root@95eefccee427:/workspace/scubaduck#I'll run a ruff check to ensure everything is in good shape.
(scubaduck) root@95eefccee427:/workspace/scubaduck# ruff check >/tmp/ruff_check.
log && tail -n 20 /tmp/ruff_check.log
All checks passed!
(scubaduck) root@95eefccee427:/workspace/scubaduck#I'll run pyright to check for type-related issues in the code.
(scubaduck) root@95eefccee427:/workspace/scubaduck# pyright >/tmp/pyright.log &&
tail -n 20 /tmp/pyright.log0 errors, 0 warnings, 0 informations
(scubaduck) root@95eefccee427:/workspace/scubaduck#I'll run pytest now to check if everything is working.
(scubaduck) root@95eefccee427:/workspace/scubaduck# pytest >/tmp/pytest.log && t
ail -n 20 /tmp/pytest.log(scubaduck) root@95eefccee427:/workspace/scubaduck#I’ll take a look at the log to gather any relevant info.
(scubaduck) root@95eefccee427:/workspace/scubaduck# tail -n 20 /tmp/pytest.log
configfile: pyproject.toml
plugins: base-url-2.1.0, xdist-3.6.1, playwright-0.7.0
created: 5/5 workers
5 workers [121 items]
................................................................E....... [ 59%]
................................................. [100%]
==================================== ERRORS ====================================
__ ERROR at setup of test_timeseries_count_no_columns_numeric_time[chromium] ___
[gw2] linux -- Python 3.12.3 /workspace/scubaduck/.venv/bin/python
file /workspace/scubaduck/tests/test_web_timeseries.py, line 381
def test_timeseries_count_no_columns_numeric_time(
E fixture 'test_dataset_server_url' not found
> available fixtures: _artifacts_recorder, _pw_artifacts_folder, _verify_u
rl, base_url, browser, browser_channel, browser_context_args, browser_name, brow
ser_type, browser_type_launch_args, cache, capfd, capfdbinary, caplog, capsys, c
apsysbinary, connect_options, context, delete_output_dir, device, doctest_namesp
ace, is_chromium, is_firefox, is_webkit, launch_browser, monkeypatch, new_contex
t, output_path, page, playwright, pytestconfig, record_property, record_testsuit
e_property, record_xml_attribute, recwarn, server_url, testrun_uid, tmp_path, tm
p_path_factory, tmpdir, tmpdir_factory, worker_id
> use 'pytest --fixtures [testpath]' for help on them.
/workspace/scubaduck/tests/test_web_timeseries.py:381
=========================== short test summary info ============================
ERROR tests/test_web_timeseries.py::test_timeseries_count_no_columns_numeric_tim
e[chromium]
======================== 120 passed, 1 error in 14.86s =========================
(scubaduck) root@95eefccee427:/workspace/scubaduck#I'll check test_web_timeseries.py to confirm if the missing fixture is already defined.
(scubaduck) root@95eefccee427:/workspace/scubaduck# grep -n "def test_dataset_se
rver_url" -n tests/test_web_timeseries.py
(scubaduck) root@95eefccee427:/workspace/scubaduck#The fixture is in a different test file, so I'll import it directly.
(scubaduck) root@95eefccee427:/workspace/scubaduck# sed -n '1,12p' tests/test_we
b_timeseries.py
from __future__ import annotations
from typing import Any
from tests.web_utils import select_value
def test_timeseries_default_query(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#graph_type", state="attached")
select_value(page, "#graph_type", "timeseries")
page.click("text=Columns")
(scubaduck) root@95eefccee427:/workspace/scubaduck#I'll add the import to access the necessary test fixture. Let's patch it now.
(scubaduck) root@95eefccee427:/workspace/scubaduck# apply_patch <<'PATCH'
> *** Begin Patch
> *** Update File: tests/test_web_timeseries.py
> @@
> -from tests.web_utils import select_value
> +from tests.web_utils import select_value
> +from tests.test_web_table import test_dataset_server_url
> *** End Patch
> PATCH
Success. Updated the following files:
M tests/test_web_timeseries.py(scubaduck) root@95eefccee427:/workspace/scubaduck# ruff format tests/test_web_t
imeseries.py tests/test_server_errors.py >/tmp/ruff_format.log && tail -n 20 /tm
p/ruff_format.log
2 files left unchanged
(scubaduck) root@95eefccee427:/workspace/scubaduck#I'll run the ruff check one more time to be thorough.
(scubaduck) root@95eefccee427:/workspace/scubaduck# ruff check >/tmp/ruff_check.
log && tail -n 20 /tmp/ruff_check.log
(scubaduck) root@95eefccee427:/workspace/scubaduck#It didn't provide output, but I'll try running pyright again to check.
tail -n 20 /tmp/pyright.log(scubaduck) root@95eefccee427:/workspace/scubaduck# cat /tmp/pyright.log
/workspace/scubaduck/tests/test_web_timeseries.py
/workspace/scubaduck/tests/test_web_timeseries.py:6:34 - error: Import "test_d
ataset_server_url" is not accessed (reportUnusedImport)
1 error, 0 warnings, 0 informations
(scubaduck) root@95eefccee427:/workspace/scubaduck#Pyright complains about an unused import. I'll replicate the fixture locally to fix this.
(scubaduck) root@95eefccee427:/workspace/scubaduck# sed -n '1,24p' tests/test_we
b_table.py
from __future__ import annotations
from typing import Any
from collections.abc import Iterator
import threading
import pytest
from werkzeug.serving import make_server
from scubaduck.server import create_app
from tests.web_utils import run_query, select_value
@pytest.fixture()
def test_dataset_server_url() -> Iterator[str]:
app = create_app("TEST")
httpd = make_server("127.0.0.1", 0, app)
port = httpd.server_port
thread = threading.Thread(target=httpd.serve_forever)
thread.start()
try:
yield f"http://127.0.0.1:{port}"
finally:
(scubaduck) root@95eefccee427:/workspace/scubaduck#I’m continuing to scroll for more information. Let's see what comes next.
eb_table.py
finally:
httpd.shutdown()
thread.join()
def test_table_sorting(page: Any, server_url: str) -> None:
run_query(
page,
server_url,
start="2024-01-01 00:00:00",
end="2024-01-03 00:00:00",
order_by="user",
order_dir="ASC",
limit=100,
)
# header alignment
align = page.evaluate(
"getComputedStyle(document.querySelector('#results th')).textAlign"
)
assert align == "left"
header = page.locator("#results th").nth(3)
def values() -> list[str]:
return page.locator("#results td:nth-child(4)").all_inner_texts()
(scubaduck) root@95eefccee427:/workspace/scubaduck#I'll check the imports at the top of test_web_timeseries.py before defining the fixture.
meseries.py
from __future__ import annotations
from typing import Any
from tests.web_utils import select_value
from tests.test_web_table import test_dataset_server_url
def test_timeseries_default_query(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#graph_type", state="attached")
select_value(page, "#graph_type", "timeseries")
page.click("text=Columns")
page.check("#column_groups input[value='value']")
page.click("text=View Settings")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
data = page.evaluate("window.lastResults")
assert "error" not in data
(scubaduck) root@95eefccee427:/workspace/scubaduck#I'll remove the fixture import and define one myself.
> *** Begin Patch
> *** Update File: tests/test_web_timeseries.py
> @@
> -from tests.web_utils import select_value
> -from tests.test_web_table import test_dataset_server_url
> +from tests.web_utils import select_value
> +from collections.abc import Iterator
> +import threading
> +import pytest
> +from werkzeug.serving import make_server
> +from scubaduck.server import create_app
> +
> +
> +@pytest.fixture()
> +def test_dataset_server_url() -> Iterator[str]:
> + app = create_app("TEST")
> + httpd = make_server("127.0.0.1", 0, app)
> + port = httpd.server_port
> + thread = threading.Thread(target=httpd.serve_forever)
> + thread.start()
> + try:
> + yield f"http://127.0.0.1:{port}"
> + finally:
> + httpd.shutdown()
> + thread.join()
> *** End Patch
> PATCH
Success. Updated the following files:
M tests/test_web_timeseries.py(scubaduck) root@95eefccee427:/workspace/scubaduck# ruff format tests/test_web_t
imeseries.py tests/test_server_errors.py >/tmp/ruff_format.log && tail -n 20 /tm
p/ruff_format.log
2 files left unchangedlog && tail -n 20 /tmp/ruff_check.log
All checks passed!
(scubaduck) root@95eefccee427:/workspace/scubaduck#Let's run pyright to check for type issues in the code.
(scubaduck) root@95eefccee427:/workspace/scubaduck# pyright >/tmp/pyright.log &&
tail -n 20 /tmp/pyright.log0 errors, 0 warnings, 0 informations
(scubaduck) root@95eefccee427:/workspace/scubaduck#I’ll try running pytest again to see if my changes work now.
(scubaduck) root@95eefccee427:/workspace/scubaduck# pytest >/tmp/pytest.log && t
ail -n 20 /tmp/pytest.log============================= test session starts ==============================
platform linux -- Python 3.12.3, pytest-8.3.5, pluggy-1.6.0
rootdir: /workspace/scubaduck
configfile: pyproject.toml
plugins: base-url-2.1.0, xdist-3.6.1, playwright-0.7.0
created: 5/5 workers
5 workers [121 items]
........................................................................ [ 59%]
................................................. [100%]
============================= 121 passed in 14.83s =============================
(scubaduck) root@95eefccee427:/workspace/scubaduck#Tests are passing! Time to check the git status and commit my changes.
(scubaduck) root@95eefccee427:/workspace/scubaduck# git status --short
M scubaduck/static/js/timeseries_chart.js
M tests/test_server_errors.py
M tests/test_web_timeseries.py
(scubaduck) root@95eefccee427:/workspace/scubaduck#I'll add some new files and check the differences to review the changes.
(scubaduck) root@95eefccee427:/workspace/scubaduck# git diff --stat
scubaduck/static/js/timeseries_chart.js | 8 ++++++-
tests/test_server_errors.py | 2 +-
tests/test_web_timeseries.py | 39 +++++++++++++++++++++++++++++++++
3 files changed, 47 insertions(+), 2 deletions(-)
(scubaduck) root@95eefccee427:/workspace/scubaduck#I'll review the differences for each component.
(scubaduck) root@95eefccee427:/workspace/scubaduck# git diff scubaduck/static/js
/timeseries_chart.js | sed -n '1,60p'
diff --git a/scubaduck/static/js/timeseries_chart.js b/scubaduck/static/js/times
eries_chart.js
index de2d658..a054468 100644
--- a/scubaduck/static/js/timeseries_chart.js
+++ b/scubaduck/static/js/timeseries_chart.js
@@ -35,7 +35,13 @@ function showTimeSeries(data) {
const start = data.start ? parseTs(data.start) : null;
const end = data.end ? parseTs(data.end) : null;
const startIdx = 1 + groups.length + hasHits;
- const valueCols = selectedColumns.slice(groups.length + hasHits);
+ let valueCols = selectedColumns.slice(groups.length + hasHits);
+ if (
+ valueCols.length === 0 &&
+ document.getElementById('aggregate').value.toLowerCase() === 'count'
+ ) {
+ valueCols = ['Count'];
+ }
const series = {};
data.rows.forEach(r => {
const ts = parseTs(r[0]);(scubaduck) root@95eefccee427:/workspace/scubaduck# git diff tests/test_server_e
rrors.py | sed -n '1,40p'
diff --git a/tests/test_server_errors.py b/tests/test_server_errors.py
index a19c554..91e3be4 100644
--- a/tests/test_server_errors.py
+++ b/tests/test_server_errors.py
@@ -62,7 +62,7 @@ def test_table_unknown_column_error() -> None:
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
- data = rv.get_json()
+ _data = rv.get_json()
assert rv.status_code == 200
(scubaduck) root@95eefccee427:/workspace/scubaduck#I’ll review the diff around the new changes in test_web_timeseries.py.
index a7ecc7c..2f3f937 100644
--- a/tests/test_web_timeseries.py
+++ b/tests/test_web_timeseries.py
@@ -3,6 +3,25 @@ from __future__ import annotations
from typing import Any
from tests.web_utils import select_value
+from collections.abc import Iterator
+import threading
+import pytest
+from werkzeug.serving import make_server
+from scubaduck.server import create_app
+
+
+@pytest.fixture()
+def test_dataset_server_url() -> Iterator[str]:
+ app = create_app("TEST")
+ httpd = make_server("127.0.0.1", 0, app)
+ port = httpd.server_port
+ thread = threading.Thread(target=httpd.serve_forever)
+ thread.start()
+ try:
+ yield f"http://127.0.0.1:{port}"
+ finally:
+ httpd.shutdown()
+ thread.join()
def test_timeseries_default_query(page: Any, server_url: str) -> None:
@@ -376,3 +395,23 @@ def test_timeseries_group_links(page: Any, server_url: str)
-> None:
assert chips == ["user"]
assert page.text_content("#legend .drill-links h4") == "Drill up"
assert page.is_visible("#legend .drill-links a:text('Aggregate')")
+
+
+def test_timeseries_count_no_columns_numeric_time(
+ page: Any, test_dataset_server_url: str
+) -> None:
+ page.goto(test_dataset_server_url)
+ page.wait_for_selector("#graph_type", state="attached")
+ select_value(page, "#graph_type", "timeseries")
+ page.click("text=Columns")
+ page.click("#columns_none")
+ page.click("text=View Settings")
+ select_value(page, "#time_column", "ts")
+ select_value(page, "#time_unit", "s")
+ select_value(page, "#aggregate", "Count")
+ page.evaluate("window.lastResults = undefined")
+ page.click("text=Dive")
+ page.wait_for_function("window.lastResults !== undefined")
+ page.wait_for_selector("#chart path", state="attached")
+ series_count = page.eval_on_selector_all("#chart path", "els => els.length"
)
+ assert series_count == 1
(scubaduck) root@95eefccee427:/workspace/scubaduck#The update is ready. Time to commit!
(scubaduck) root@95eefccee427:/workspace/scubaduck# git add scubaduck/static/js/
timeseries_chart.js tests/test_server_errors.py tests/test_web_timeseries.pycount series for numeric timestamps" && git status --short
[work b371311] fix(ui): show count series for numeric timestamps
3 files changed, 47 insertions(+), 2 deletions(-)
(scubaduck) root@95eefccee427:/workspace/scubaduck#It looks like the output was cut off. I'll review the commit message.
(scubaduck) root@95eefccee427:/workspace/scubaduck# git log -1 --stat
commit b371311b8e1fc9cccf6104a7dc9fb87cf5bda36a (HEAD -> work)
Author: Codex <codex@openai.com>
Date: Fri May 23 02:34:18 2025 +0000
fix(ui): show count series for numeric timestamps
scubaduck/static/js/timeseries_chart.js | 8 ++++++-
tests/test_server_errors.py | 2 +-
tests/test_web_timeseries.py | 39 +++++++++++++++++++++++++++++++++
3 files changed, 47 insertions(+), 2 deletions(-)(scubaduck) root@95eefccee427:/workspace/scubaduck# git status --short
(scubaduck) root@95eefccee427:/workspace/scubaduck#I’ll capture the lines from our changes in timeseries_chart.js for reference.
(scubaduck) root@95eefccee427:/workspace/scubaduck# nl -ba scubaduck/static/js/t
imeseries_chart.js | sed -n '30,45p'
30 crosshairDots.style.display = 'none';
31 const groups = groupBy.chips || [];
32 const hasHits = document.getElementById('show_hits').checked ? 1 : 0;
33 const fill = document.getElementById('fill').value;
34 const bucketMs = (data.bucket_size || 3600) * 1000;
35 const start = data.start ? parseTs(data.start) : null;
36 const end = data.end ? parseTs(data.end) : null;
37 const startIdx = 1 + groups.length + hasHits;
38 let valueCols = selectedColumns.slice(groups.length + hasHits);
39 if (
40 valueCols.length === 0 &&
41 document.getElementById('aggregate').value.toLowerCase() === 'count'
42 ) {
43 valueCols = ['Count'];
44 }
45 const series = {};(scubaduck) root@95eefccee427:/workspace/scubaduck# nl -ba tests/test_web_timese
ries.py | sed -n '1,26p'
1 from __future__ import annotations
2
3 from typing import Any
4
5 from tests.web_utils import select_value
6 from collections.abc import Iterator
7 import threading
8 import pytest
9 from werkzeug.serving import make_server
10 from scubaduck.server import create_app
11
12
13 @pytest.fixture()
14 def test_dataset_server_url() -> Iterator[str]:
15 app = create_app("TEST")
16 httpd = make_server("127.0.0.1", 0, app)
17 port = httpd.server_port
18 thread = threading.Thread(target=httpd.serve_forever)
19 thread.start()
20 try:
21 yield f"http://127.0.0.1:{port}"
22 finally:
23 httpd.shutdown()
24 thread.join()
25
26(scubaduck) root@95eefccee427:/workspace/scubaduck# nl -ba tests/test_web_timese
ries.py | sed -n '390,412p'
390 page.wait_for_function("window.lastResults !== undefined")
391 assert page.text_content("#legend .drill-links h4") == "Group by"
392 page.click("#legend .drill-links a:text('user')")
393 page.wait_for_function("window.lastResults !== undefined")
394 chips = page.evaluate("groupBy.chips")
395 assert chips == ["user"]
396 assert page.text_content("#legend .drill-links h4") == "Drill up"
397 assert page.is_visible("#legend .drill-links a:text('Aggregate')")
398
399
400 def test_timeseries_count_no_columns_numeric_time(
401 page: Any, test_dataset_server_url: str
402 ) -> None:
403 page.goto(test_dataset_server_url)
404 page.wait_for_selector("#graph_type", state="attached")
405 select_value(page, "#graph_type", "timeseries")
406 page.click("text=Columns")
407 page.click("#columns_none")
408 page.click("text=View Settings")
409 select_value(page, "#time_column", "ts")
410 select_value(page, "#time_unit", "s")
411 select_value(page, "#aggregate", "Count")
412 page.evaluate("window.lastResults = undefined")(scubaduck) root@95eefccee427:/workspace/scubaduck# nl -ba tests/test_web_timese
ries.py | sed -n '412,430p'
412 page.evaluate("window.lastResults = undefined")
413 page.click("text=Dive")
414 page.wait_for_function("window.lastResults !== undefined")
415 page.wait_for_selector("#chart path", state="attached")
416 series_count = page.eval_on_selector_all("#chart path", "els => els.
length")
417 assert series_count == 1(scubaduck) root@95eefccee427:/workspace/scubaduck# nl -ba tests/test_server_err
ors.py | sed -n '60,66p'
60 "show_hits": True,
61 }
62 rv = client.post(
63 "/api/query", data=json.dumps(payload), content_type="applicatio
n/json"
64 )
65 _data = rv.get_json()
66 assert rv.status_code == 200
(scubaduck) root@95eefccee427:/workspace/scubaduck#I’ll draft the PR message, summarizing everything and referencing the important lines.