Task ID: task_e_682fc90c1b388323b0f480e12882e0cc
Configuring language runtimes...
+ rm -- /tmp/oNa6ki-setup_script.sh
+ source ./codex_setup.sh
++ uv sync --frozen
Using CPython 3.12.3 interpreter at: /usr/bin/python3.12
Creating virtual environment at: .venv
Building scubaduck @ file:///workspace/scubaduck
Downloading ruff (11.0MiB)
Downloading pyright (5.3MiB)
Downloading duckdb (19.3MiB)
Downloading playwright (43.1MiB)
Downloading ruff
Built scubaduck @ file:///workspace/scubaduck
Downloading duckdb
Downloading playwright
Downloading pyright
Prepared 33 packages in 1.56s
Installed 33 packages in 75ms
+ blinker==1.9.0
+ certifi==2025.4.26
+ charset-normalizer==3.4.2
+ click==8.2.0
+ duckdb==1.2.2
+ execnet==2.1.1
+ flask==3.1.1
+ greenlet==3.2.2
+ idna==3.10
+ iniconfig==2.1.0
+ itsdangerous==2.2.0
+ jinja2==3.1.6
+ markupsafe==3.0.2
+ nodeenv==1.9.1
+ packaging==25.0
+ playwright==1.52.0
+ pluggy==1.6.0
+ pyee==13.0.0
+ pyright==1.1.400
+ pytest==8.3.5
+ pytest-base-url==2.1.0
+ pytest-playwright==0.7.0
+ pytest-xdist==3.6.1
+ python-dateutil==2.9.0.post0
+ python-slugify==8.0.4
+ requests==2.32.3
+ ruff==0.11.10
+ scubaduck==0.1.0 (from file:///workspace/scubaduck)
+ six==1.17.0
+ text-unidecode==1.3
+ typing-extensions==4.13.2
+ urllib3==2.4.0
+ werkzeug==3.1.3
++ source .venv/bin/activate
+++ '[' -z '' ']'
+++ '[' -n x ']'
+++ SCRIPT_PATH=.venv/bin/activate
+++ '[' .venv/bin/activate = /tmp/oNa6ki-setup_script.sh ']'
+++ deactivate nondestructive
+++ unset -f pydoc
+++ '[' -z '' ']'
+++ '[' -z '' ']'
+++ hash -r
+++ '[' -z '' ']'
+++ unset VIRTUAL_ENV
+++ unset VIRTUAL_ENV_PROMPT
+++ '[' '!' nondestructive = nondestructive ']'
+++ VIRTUAL_ENV=/workspace/scubaduck/.venv
+++ '[' linux-gnu = cygwin ']'
+++ '[' linux-gnu = msys ']'
+++ export VIRTUAL_ENV
+++ '[' -z '' ']'
+++ unset SCRIPT_PATH
+++ _OLD_VIRTUAL_PATH=/root/.cargo/bin:/root/.rbenv/shims:/root/.rbenv/bin:/root/.rbenv/shims:/root/.local/share/swiftly/bin:/root/.bun/bin:/root/.nvm/versions/node/v22.16.0/bin:/root/.pyenv/shims:3441PYENV_ROOT/shims:/root/.pyenv/bin:/usr/local/go/bin:/root/go/bin:/root/.rbenv/bin:/root/.rbenv/shims:/root/.bun/bin:/root/.local/bin:/root/.pyenv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+++ PATH=/workspace/scubaduck/.venv/bin:/root/.cargo/bin:/root/.rbenv/shims:/root/.rbenv/bin:/root/.rbenv/shims:/root/.local/share/swiftly/bin:/root/.bun/bin:/root/.nvm/versions/node/v22.16.0/bin:/root/.pyenv/shims:3441PYENV_ROOT/shims:/root/.pyenv/bin:/usr/local/go/bin:/root/go/bin:/root/.rbenv/bin:/root/.rbenv/shims:/root/.bun/bin:/root/.local/bin:/root/.pyenv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+++ export PATH
+++ '[' xscubaduck '!=' x ']'
+++ VIRTUAL_ENV_PROMPT=scubaduck
+++ export VIRTUAL_ENV_PROMPT
+++ '[' -z '' ']'
+++ '[' -z '' ']'
+++ _OLD_VIRTUAL_PS1=
+++ PS1='(scubaduck) '
+++ export PS1
+++ alias pydoc
+++ true
+++ hash -r
++ python -c 'import os; import duckdb; con = duckdb.connect(); con.execute(f"SET http_proxy = '\''{os.getenv("HTTP_PROXY")}'\''"); con.execute("INSTALL '\''sqlite'\'';")'
++ playwright install chromium
Downloading Chromium 136.0.7103.25 (playwright build v1169) from https://cdn.playwright.dev/dbazure/download/playwright/builds/chromium/1169/chromium-linux.zip
167.7 MiB [] 0% 0.0s167.7 MiB [] 0% 17.2s167.7 MiB [] 0% 12.6s167.7 MiB [] 0% 8.8s167.7 MiB [] 1% 5.1s167.7 MiB [] 2% 4.1s167.7 MiB [] 3% 3.0s167.7 MiB [] 4% 2.5s167.7 MiB [] 5% 2.3s167.7 MiB [] 6% 2.4s167.7 MiB [] 7% 2.2s167.7 MiB [] 9% 1.9s167.7 MiB [] 10% 1.8s167.7 MiB [] 12% 1.6s167.7 MiB [] 14% 1.4s167.7 MiB [] 16% 1.3s167.7 MiB [] 18% 1.2s167.7 MiB [] 20% 1.1s167.7 MiB [] 22% 1.0s167.7 MiB [] 24% 1.0s167.7 MiB [] 26% 0.9s167.7 MiB [] 28% 0.9s167.7 MiB [] 29% 0.9s167.7 MiB [] 32% 0.8s167.7 MiB [] 34% 0.8s167.7 MiB [] 37% 0.7s167.7 MiB [] 38% 0.7s167.7 MiB [] 40% 0.6s167.7 MiB [] 42% 0.6s167.7 MiB [] 44% 0.6s167.7 MiB [] 46% 0.6s167.7 MiB [] 47% 0.6s167.7 MiB [] 49% 0.5s167.7 MiB [] 51% 0.5s167.7 MiB [] 53% 0.5s167.7 MiB [] 55% 0.5s167.7 MiB [] 57% 0.4s167.7 MiB [] 58% 0.4s167.7 MiB [] 59% 0.4s167.7 MiB [] 61% 0.4s167.7 MiB [] 62% 0.4s167.7 MiB [] 65% 0.4s167.7 MiB [] 67% 0.3s167.7 MiB [] 69% 0.3s167.7 MiB [] 70% 0.3s167.7 MiB [] 71% 0.3s167.7 MiB [] 73% 0.3s167.7 MiB [] 75% 0.2s167.7 MiB [] 77% 0.2s167.7 MiB [] 79% 0.2s167.7 MiB [] 80% 0.2s167.7 MiB [] 82% 0.2s167.7 MiB [] 85% 0.2s167.7 MiB [] 87% 0.1s167.7 MiB [] 89% 0.1s167.7 MiB [] 91% 0.1s167.7 MiB [] 93% 0.1s167.7 MiB [] 95% 0.1s167.7 MiB [] 96% 0.0s167.7 MiB [] 97% 0.0s167.7 MiB [] 99% 0.0s167.7 MiB [] 100% 0.0s
Chromium 136.0.7103.25 (playwright build v1169) downloaded to /root/.cache/ms-playwright/chromium-1169
Downloading FFMPEG playwright build v1011 from https://cdn.playwright.dev/dbazure/download/playwright/builds/ffmpeg/1011/ffmpeg-linux.zip
2.3 MiB [] 0% 0.0s2.3 MiB [] 5% 0.3s2.3 MiB [] 19% 0.1s2.3 MiB [] 50% 0.0s2.3 MiB [] 89% 0.0s2.3 MiB [] 100% 0.0s
FFMPEG playwright build v1011 downloaded to /root/.cache/ms-playwright/ffmpeg-1011
Downloading Chromium Headless Shell 136.0.7103.25 (playwright build v1169) from https://cdn.playwright.dev/dbazure/download/playwright/builds/chromium/1169/chromium-headless-shell-linux.zip
101.4 MiB [] 0% 0.0s101.4 MiB [] 0% 18.4s101.4 MiB [] 0% 12.3s101.4 MiB [] 0% 9.4s101.4 MiB [] 1% 6.1s101.4 MiB [] 2% 3.9s101.4 MiB [] 4% 2.4s101.4 MiB [] 6% 1.6s101.4 MiB [] 8% 1.4s101.4 MiB [] 11% 1.2s101.4 MiB [] 13% 1.1s101.4 MiB [] 16% 0.9s101.4 MiB [] 19% 0.8s101.4 MiB [] 21% 0.8s101.4 MiB [] 22% 0.8s101.4 MiB [] 25% 0.7s101.4 MiB [] 27% 0.7s101.4 MiB [] 28% 0.7s101.4 MiB [] 31% 0.6s101.4 MiB [] 34% 0.6s101.4 MiB [] 37% 0.6s101.4 MiB [] 39% 0.5s101.4 MiB [] 42% 0.5s101.4 MiB [] 45% 0.5s101.4 MiB [] 48% 0.4s101.4 MiB [] 50% 0.4s101.4 MiB [] 54% 0.4s101.4 MiB [] 57% 0.3s101.4 MiB [] 59% 0.3s101.4 MiB [] 63% 0.3s101.4 MiB [] 66% 0.3s101.4 MiB [] 67% 0.3s101.4 MiB [] 70% 0.2s101.4 MiB [] 73% 0.2s101.4 MiB [] 75% 0.2s101.4 MiB [] 77% 0.2s101.4 MiB [] 80% 0.2s101.4 MiB [] 83% 0.1s101.4 MiB [] 86% 0.1s101.4 MiB [] 87% 0.1s101.4 MiB [] 90% 0.1s101.4 MiB [] 92% 0.1s101.4 MiB [] 95% 0.0s101.4 MiB [] 98% 0.0s101.4 MiB [] 99% 0.0s101.4 MiB [] 100% 0.0s
Chromium Headless Shell 136.0.7103.25 (playwright build v1169) downloaded to /root/.cache/ms-playwright/chromium_headless_shell-1169
++ echo 'source .venv/bin/activate'ls^Clsls
bash-5.2#
bash-5.2# ls
scubaduck
bash-5.2#I'm thinking the terminal might be at root—maybe I triggered something by hitting ^L.
bash: .venv/bin/activate: No such file or directory
root@eb913c2329bf:/workspace# ls
scubaduckThe shell error is odd. I'll check the scubaduck repo for any issues.
root@eb913c2329bf:/workspace# cd scubaduck
root@eb913c2329bf:/workspace/scubaduck# ls
AGENTS.md codex_setup.sh scubaduck tests
README.md pyproject.toml stubs uv.lockI found AGENTS.md at the root of the repo. Time to check it out!
root@eb913c2329bf:/workspace/scubaduck# cat AGENTS.md
- Use "source .venv/bin/activate" to activate the virtual environment before
running any commands
- ONLY use non-interactive commands like cat, sed, apply_patch to do edits.
Do NOT use interactive editors.
- Do NOT attempt to install packages. Only the packages specified in
pyproject.toml are available. You cannot add new packages. If you
desperately want another package, make a note of it in the final PR
description.
- Use conventional commits to format PR title
- There are no nested AGENTS.md files, this is the only agents file
- When using Playwright, ONLY use chromium browser. This browser is already
installed.
- Use "ruff check" to check lint, "ruff format" to autoformat files and
"pyright" to typecheck.
- When writing the PR description, include the original prompt supplied by the
user.
- When you add functionality to the server, add server tests to
tests/test_server.py. When you add features to the frontend, add frontend
tests to tests/test_web.pyroot@eb913c2329bf:/workspace/scubaduck# ls
AGENTS.md codex_setup.sh scubaduck tests
README.md pyproject.toml stubs uv.lockLet's see what's inside the scubaduck directory to understand it better.
root@eb913c2329bf:/workspace/scubaduck# ls scubaduck
__init__.py sample.csv server.py staticI’ll start by taking a look at server.py. Let’s see what’s inside!
root@eb913c2329bf:/workspace/scubaduck# sed -n '1,160p' scubaduck/server.py
from __future__ import annotations
from dataclasses import dataclass, field, replace
from typing import Any, Dict, List, Tuple, cast
import re
from datetime import datetime, timedelta, timezone
import time
from pathlib import Path
import os
import traceback
import math
import duckdb
from dateutil import parser as dtparser
from flask import Flask, jsonify, request, send_from_directory
@dataclass
class Filter:
column: str
op: str
value: str | int | float | list[str] | None
@dataclass
class QueryParams:
start: str | None = None
end: str | None = None
order_by: str | None = None
order_dir: str = "ASC"
limit: int | None = None
columns: list[str] = field(default_factory=lambda: [])
filters: list[Filter] = field(default_factory=lambda: [])
derived_columns: dict[str, str] = field(default_factory=lambda: {})
graph_type: str = "samples"
group_by: list[str] = field(default_factory=lambda: [])
aggregate: str | None = None
show_hits: bool = False
x_axis: str | None = None
granularity: str = "Auto"
fill: str = "0"
table: str = "events"
time_column: str | None = "timestamp"
time_unit: str = "s"
def _load_database(path: Path) -> duckdb.DuckDBPyConnection:
if not path.exists():
raise FileNotFoundError(path)
ext = path.suffix.lower()
if ext == ".csv":
con = duckdb.connect()
con.execute(
f"CREATE TABLE events AS SELECT * FROM read_csv_auto('{path.as_posix
()}')"
)
elif ext in {".db", ".sqlite"}:
con = duckdb.connect()
con.execute("LOAD sqlite")
con.execute(f"ATTACH '{path.as_posix()}' AS db (TYPE SQLITE)")
tables = [
r[0]
for r in con.execute(
"SELECT name FROM sqlite_master WHERE type='table'"
).fetchall()
]
for t in tables:
con.execute(f'CREATE VIEW "{t}" AS SELECT * FROM db."{t}"')
else:
con = duckdb.connect(path)
return con
def _create_test_database() -> duckdb.DuckDBPyConnection:
"""Return a DuckDB connection with a small multi-table dataset."""
con = duckdb.connect()
con.execute(
"CREATE TABLE events (id INTEGER PRIMARY KEY, ts TEXT, val REAL, name TE
XT, flag BOOLEAN)"
)
con.execute("INSERT INTO events VALUES (1, '2024-01-01 00:00:00', 1.5, 'alic
e', 1)")
con.execute("INSERT INTO events VALUES (2, '2024-01-01 01:00:00', 2.0, 'bob'
, 0)")
con.execute('CREATE TABLE extra (ts TEXT, "desc" TEXT, num INTEGER)')
con.execute("INSERT INTO extra VALUES ('2024-01-01 00:00:00', 'x', 1)")
con.execute("INSERT INTO extra VALUES ('2024-01-01 01:00:00', 'y', 2)")
return con
_REL_RE = re.compile(
r"([+-]?\d+(?:\.\d*)?)\s*(hour|hours|day|days|week|weeks|fortnight|fortnight
s)",
re.IGNORECASE,
)
def parse_time(val: str | None) -> str | None:
"""Parse an absolute or relative time string into ``YYYY-MM-DD HH:MM:SS``.""
"
if val is None or val == "":
return None
s = val.strip()
if s.lower() == "now":
dt = datetime.now(timezone.utc)
return dt.replace(microsecond=0).strftime("%Y-%m-%d %H:%M:%S")
m = _REL_RE.fullmatch(s)
if m:
qty = float(m.group(1))
unit = m.group(2).lower()
delta: timedelta
if unit.startswith("hour"):
delta = timedelta(hours=qty)
elif unit.startswith("day"):
delta = timedelta(days=qty)
elif unit.startswith("week"):
delta = timedelta(weeks=qty)
elif unit.startswith("fortnight"):
delta = timedelta(weeks=2 * qty)
else: # pragma: no cover - defensive
raise ValueError(f"Unsupported unit: {unit}")
dt = datetime.now(timezone.utc) + delta
return dt.replace(microsecond=0).strftime("%Y-%m-%d %H:%M:%S")
dt = dtparser.parse(s)
return dt.replace(microsecond=0, tzinfo=None).strftime("%Y-%m-%d %H:%M:%S")
def _granularity_seconds(granularity: str, start: str | None, end: str | None) -
> int:
gran = granularity.lower()
mapping = {
"1 second": 1,
"5 seconds": 5,
"10 seconds": 10,
"30 seconds": 30,
"1 minute": 60,
"4 minutes": 240,
"5 minutes": 300,
"10 minutes": 600,
"15 minutes": 900,
"30 minutes": 1800,
"1 hour": 3600,
"3 hours": 10800,
"6 hours": 21600,
"1 day": 86400,
"1 week": 604800,
"30 days": 2592000,
}
if gran in mapping:
return mapping[gran]
if gran in {"auto", "fine"} and start and end:
try:
s = dtparser.parse(start)
e = dtparser.parse(end)
except Exception:
return 3600
total = max((e - s).total_seconds(), 1)
buckets = 100 if gran == "auto" else 500
return max(int(total // buckets), 1)
return 3600root@eb913c2329bf:/workspace/scubaduck# sed -n '160,320p' scubaduck/server.py
def _time_expr(col: str, column_types: Dict[str, str] | None, unit: str) -> str:
"""Return SQL expression for column interpreted as timestamp."""
if column_types is None:
return col
ctype = column_types.get(col, "").upper()
if not any(t in ctype for t in ["TIMESTAMP", "DATE", "TIME"]):
if any(
t in ctype
for t in [
"INT",
"DECIMAL",
"REAL",
"DOUBLE",
"FLOAT",
"NUMERIC",
"HUGEINT",
]
):
if unit == "ns":
expr = f"CAST({col} AS BIGINT)"
return f"make_timestamp_ns({expr})"
multiplier = {
"s": 1_000_000,
"ms": 1_000,
"us": 1,
}.get(unit, 1_000_000)
expr = (
f"CAST({col} * {multiplier} AS BIGINT)"
if multiplier != 1
else f"CAST({col} AS BIGINT)"
)
return f"make_timestamp({expr})"
return col
def build_query(params: QueryParams, column_types: Dict[str, str] | None = None)
-> str:
select_parts: list[str] = []
group_cols = params.group_by[:]
if params.graph_type == "timeseries":
sec = _granularity_seconds(params.granularity, params.start, params.end)
x_axis = params.x_axis or params.time_column
if x_axis is None:
raise ValueError("x_axis required for timeseries")
xexpr = _time_expr(x_axis, column_types, params.time_unit)
if params.start:
bucket_expr = (
f"TIMESTAMP '{params.start}' + INTERVAL '{sec} second' * "
f"CAST(floor((epoch({xexpr}) - epoch(TIMESTAMP '{params.start}')
)/{sec}) AS BIGINT)"
)
else:
bucket_expr = (
f"TIMESTAMP 'epoch' + INTERVAL '{sec} second' * "
f"CAST(floor(epoch({xexpr})/{sec}) AS BIGINT)"
)
select_parts.append(f"{bucket_expr} AS bucket")
group_cols = ["bucket"] + group_cols
has_agg = bool(group_cols) or params.aggregate is not None
if has_agg:
select_cols = (
group_cols[1:] if params.graph_type == "timeseries" else group_cols
)
select_parts.extend(select_cols)
agg = (params.aggregate or "count").lower()
def agg_expr(col: str) -> str:
expr = col
ctype = column_types.get(col, "").upper() if column_types else ""
if "BOOL" in ctype:
expr = f"CAST({col} AS BIGINT)"
if agg.startswith("p"):
quant = float(agg[1:]) / 100
return f"quantile({expr}, {quant})"
if agg == "count distinct":
return f"count(DISTINCT {expr})"
if agg == "avg" and column_types is not None:
if "TIMESTAMP" in ctype or "DATE" in ctype or "TIME" in ctype:
return (
"TIMESTAMP 'epoch' + INTERVAL '1 second' * "
f"CAST(avg(epoch({col})) AS BIGINT)"
)
return f"{agg}({expr})"
if agg == "count":
select_parts.append("count(*) AS Count")
else:
for col in params.columns:
if col in group_cols:
continue
select_parts.append(f"{agg_expr(col)} AS {col}")
if params.show_hits:
select_parts.insert(len(group_cols), "count(*) AS Hits")
else:
select_parts.extend(params.columns)
if has_agg and params.derived_columns:
inner_params = replace(
params,
derived_columns={},
order_by=None,
limit=None,
)
inner_sql = build_query(inner_params, column_types)
outer_select = ["t.*"] + [
f"{expr} AS {name}" for name, expr in params.derived_columns.items()
]
indented_inner = "\n".join(" " + line for line in inner_sql.splitline
s())
lines = [
f"SELECT {', '.join(outer_select)}",
"FROM (",
indented_inner,
") t",
]
if params.order_by:
lines.append(f"ORDER BY {params.order_by} {params.order_dir}")
elif params.graph_type == "timeseries":
lines.append("ORDER BY bucket")
if params.limit is not None:
lines.append(f"LIMIT {params.limit}")
return "\n".join(lines)
for name, expr in params.derived_columns.items():
select_parts.append(f"{expr} AS {name}")
select_clause = ", ".join(select_parts) if select_parts else "*"
lines = [f"SELECT {select_clause}", f'FROM "{params.table}"']
where_parts: list[str] = []
if params.time_column:
time_expr = _time_expr(params.time_column, column_types, params.time_uni
t)
else:
time_expr = None
if time_expr and params.start:
where_parts.append(f"{time_expr} >= '{params.start}'")
if time_expr and params.end:
where_parts.append(f"{time_expr} <= '{params.end}'")
for f in params.filters:
op = f.op
if op in {"empty", "!empty"}:
val = "''"
else:
if f.value is None:
continue
if isinstance(f.value, list):
if not f.value:
continue
if op == "=":
vals = " OR ".join(f"{f.column} = '{v}'" for v in f.value)
where_parts.append(f"({vals})")
continue
val = f"'{f.value}'" if isinstance(f.value, str) else str(f.value)
if op == "contains":
where_parts.append(f"{f.column} ILIKE '%' || {val} || '%'")
elif op == "!contains":
where_parts.append(f"{f.column} NOT ILIKE '%' || {val} || '%'")
elif op == "empty":
where_parts.append(f"{f.column} = {val}")
elif op == "!empty":
where_parts.append(f"{f.column} != {val}")
else:
where_parts.append(f"{f.column} {op} {val}")root@eb913c2329bf:/workspace/scubaduck# sed -n '320,640p' scubaduck/server.py
where_parts.append(f"{f.column} {op} {val}")
if where_parts:
lines.append("WHERE " + " AND ".join(where_parts))
if group_cols:
lines.append("GROUP BY " + ", ".join(group_cols))
if params.order_by:
lines.append(f"ORDER BY {params.order_by} {params.order_dir}")
elif params.graph_type == "timeseries":
lines.append("ORDER BY bucket")
if params.limit is not None:
lines.append(f"LIMIT {params.limit}")
return "\n".join(lines)
def create_app(db_file: str | Path | None = None) -> Flask:
app = Flask(__name__, static_folder="static")
if db_file is None:
env_db = os.environ.get("SCUBADUCK_DB")
if env_db:
db_file = env_db
if isinstance(db_file, str) and db_file.upper() == "TEST":
con = _create_test_database()
else:
db_path = Path(db_file or Path(__file__).with_name("sample.csv")).resolv
e()
con = _load_database(db_path)
tables = [r[0] for r in con.execute("SHOW TABLES").fetchall()]
if not tables:
raise ValueError("No tables found in database")
default_table = tables[0]
columns_cache: Dict[str, Dict[str, str]] = {}
def get_columns(table: str) -> Dict[str, str]:
if table not in columns_cache:
rows = con.execute(f'PRAGMA table_info("{table}")').fetchall()
if not rows:
raise ValueError(f"Unknown table: {table}")
columns_cache[table] = {r[1]: r[2] for r in rows}
return columns_cache[table]
sample_cache: Dict[Tuple[str, str, str], Tuple[List[str], float]] = {}
CACHE_TTL = 60.0
CACHE_LIMIT = 200
@app.route("/")
def index() -> Any: # pyright: ignore[reportUnusedFunction]
assert app.static_folder is not None
return send_from_directory(app.static_folder, "index.html")
@app.route("/js/<path:filename>")
def js(filename: str) -> Any: # pyright: ignore[reportUnusedFunction]
assert app.static_folder is not None
folder = Path(app.static_folder) / "js"
return send_from_directory(folder, filename)
@app.route("/api/tables")
def tables_endpoint() -> Any: # pyright: ignore[reportUnusedFunction]
return jsonify(tables)
@app.route("/api/columns")
def columns() -> Any: # pyright: ignore[reportUnusedFunction]
table = request.args.get("table", default_table)
rows = con.execute(f'PRAGMA table_info("{table}")').fetchall()
return jsonify([{"name": r[1], "type": r[2]} for r in rows])
def _cache_get(key: Tuple[str, str, str]) -> List[str] | None:
item = sample_cache.get(key)
if item is None:
return None
vals, ts = item
if time.time() - ts > CACHE_TTL:
del sample_cache[key]
return None
sample_cache[key] = (vals, time.time())
return vals
def _cache_set(key: Tuple[str, str, str], vals: List[str]) -> None:
sample_cache[key] = (vals, time.time())
if len(sample_cache) > CACHE_LIMIT:
oldest = min(sample_cache.items(), key=lambda kv: kv[1][1])[0]
del sample_cache[oldest]
@app.route("/api/samples")
def sample_values() -> Any: # pyright: ignore[reportUnusedFunction]
table = request.args.get("table", default_table)
column = request.args.get("column")
substr = request.args.get("q", "")
column_types = get_columns(table)
if not column or column not in column_types:
return jsonify([])
ctype = column_types[column].upper()
if "CHAR" not in ctype and "STRING" not in ctype and "VARCHAR" not in ct
ype:
return jsonify([])
key = (table, column, substr)
cached = _cache_get(key)
if cached is not None:
return jsonify(cached)
rows = con.execute(
f"SELECT DISTINCT {column} FROM \"{table}\" WHERE CAST({column} AS V
ARCHAR) ILIKE '%' || ? || '%' LIMIT 20",
[substr],
).fetchall()
values = [r[0] for r in rows]
_cache_set(key, values)
return jsonify(values)
@app.route("/api/query", methods=["POST"])
def query() -> Any: # pyright: ignore[reportUnusedFunction]
payload = request.get_json(force=True)
try:
start = parse_time(payload.get("start"))
end = parse_time(payload.get("end"))
except Exception as exc:
return jsonify({"error": str(exc)}), 400
params = QueryParams(
start=start,
end=end,
order_by=payload.get("order_by"),
order_dir=payload.get("order_dir", "ASC"),
limit=payload.get("limit"),
columns=payload.get("columns", []),
...
continue
if c not in column_types:
continue
ctype = column_types.get(c, "").upper()
is_numeric = any(
t in ctype
for t in [
"INT",
"DECIMAL",
"REAL",
"DOUBLE",
"FLOAT",
"NUMERIC",
"HUGEINT",
]
)
is_time = "TIMESTAMP" in ctype or "DATE" in ctype or "TIME"
in ctype
if need_numeric and not is_numeric:
return (
jsonify(
{
"error": f"Aggregate {agg} cannot be applied
to column {c}",
}
),
400,
)
if allow_time and not (is_numeric or is_time):
return (
jsonify(
{
"error": f"Aggregate {agg} cannot be applied
to column {c}",
}
),
400,
)
if (params.start is None or params.end is None) and (
params.x_axis or params.time_column
):
axis = params.x_axis or params.time_column
row = cast(
tuple[datetime | None, datetime | None],
con.execute(
f'SELECT min({axis}), max({axis}) FROM "{params.table}"'
).fetchall()[0],
)
mn, mx = row
divisor = {
"s": 1,
"ms": 1000,
"us": 1_000_000,
"ns": 1_000_000_000,
}.get(params.time_unit, 1)
if isinstance(mn, (int, float)):
try:
mn = datetime.fromtimestamp(int(mn) / divisor, tz=timezone.u
tc)
except Exception:
return (
jsonify(
{
"error": (
f"Invalid time value {mn} for column {axis}"
f" with time_unit {params.time_unit}"
)
}
),
400,
)
if isinstance(mx, (int, float)):
try:
mx = datetime.fromtimestamp(int(mx) / divisor, tz=timezone.u
tc)
except Exception:
return (
jsonify(
{
"error": (
f"Invalid time value {mx} for column {axis}"
f" with time_unit {params.time_unit}"
)
}
),
400,
)
if params.start is None and mn is not None:
params.start = (
mn.strftime("%Y-%m-%d %H:%M:%S") if not isinstance(mn, str)
else mn
)
if params.end is None and mx is not None:
params.end = (
mx.strftime("%Y-%m-%d %H:%M:%S") if not isinstance(mx, str)
else mx
)
bucket_size: int | None = None
if params.graph_type == "timeseries":
bucket_size = _granularity_seconds(
params.granularity,
params.start if isinstance(params.start, str) else None,
params.end if isinstance(params.end, str) else None,
)
if (
params.limit is not None
and params.start is not None
and params.end is not None
):
try:
start_dt = dtparser.parse(params.start)
end_dt = dtparser.parse(params.end)
buckets = math.ceil(
(end_dt - start_dt).total_seconds() / bucket_size
)
if buckets > 1:
params.limit *= buckets
except Exception:
pass
sql = build_query(params, column_types)
try:
rows = con.execute(sql).fetchall()root@eb913c2329bf:/workspace/scubaduck# sed -n '640,960p' scubaduck/server.py
rows = con.execute(sql).fetchall()
except Exception as exc:
tb = traceback.format_exc()
print(f"Query failed:\n{sql}\n{tb}")
return (
jsonify({"sql": sql, "error": str(exc), "traceback": tb}),
400,
)
result: Dict[str, Any] = {"sql": sql, "rows": rows}
if params.start is not None:
result["start"] = str(params.start)
if params.end is not None:
result["end"] = str(params.end)
if bucket_size is not None:
result["bucket_size"] = bucket_size
return jsonify(result)
return app
app = create_app()
if __name__ == "__main__":
app.run(debug=True)root@eb913c2329bf:/workspace/scubaduck# grep -n "desc" -n scubaduck/server.py
84: con.execute('CREATE TABLE extra (ts TEXT, "desc" TEXT, num INTEGER)')root@eb913c2329bf:/workspace/scubaduck# ls tests
conftest.py test_multi_table_web.py test_server.py test_web.pyroot@eb913c2329bf:/workspace/scubaduck# sed -n '1,160p' tests/test_server.py
from __future__ import annotations
import json
from pathlib import Path
from typing import Any, cast
import duckdb
from scubaduck import server
import pytest
def test_basic_query() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-02 00:00:00",
"order_by": "timestamp",
"order_dir": "ASC",
"limit": 10,
"columns": ["timestamp", "event", "value", "user"],
"filters": [],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert data
rows = data["rows"]
# We expect first three rows (until 2024-01-02 00:00:00)
assert len(rows) == 3
assert rows[0][1] == "login"
assert rows[1][1] == "logout"
def test_js_served() -> None:
app = server.app
client = app.test_client()
rv = client.get("/js/chip_input.js")
assert rv.status_code == 200
assert b"initChipInput" in rv.data
def test_filter_multi_token() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-02 03:00:00",
"order_by": "timestamp",
"limit": 10,
"columns": ["timestamp", "event", "value", "user"],
"filters": [{"column": "user", "op": "=", "value": ["alice", "charlie"]}
],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert data
rows = data["rows"]
# Should only return rows for alice and charlie
assert len(rows) == 3
assert rows[0][3] == "alice"
assert rows[-1][3] == "charlie"
def test_empty_filter_is_noop() -> None:
app = server.app
client = app.test_client()
base_payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"limit": 100,
"columns": ["timestamp", "event", "value", "user"],
}
no_filter = {**base_payload, "filters": []}
empty_filter = {
**base_payload,
"filters": [{"column": "user", "op": "=", "value": None}],
}
rv1 = client.post(
"/api/query", data=json.dumps(no_filter), content_type="application/json
"
)
rv2 = client.post(
"/api/query", data=json.dumps(empty_filter), content_type="application/j
son"
)
rows1 = rv1.get_json()["rows"]
rows2 = rv2.get_json()["rows"]
assert rows1 == rows2
def test_select_columns() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"order_by": "timestamp",
"limit": 10,
"columns": ["timestamp", "user"],
"filters": [],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert data
rows = data["rows"]
assert len(rows[0]) == 2
assert rows[0][1] == "alice"
def test_string_filter_ops() -> None:
app = server.app
client = app.test_client()
base = {
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"order_by": "timestamp",
"limit": 100,
"columns": ["timestamp", "event", "value", "user"],
}
contains = {
**base,
"filters": [{"column": "user", "op": "contains", "value": "ali"}],
}
rv = client.post(
"/api/query", data=json.dumps(contains), content_type="application/json"
)
rows = rv.get_json()["rows"]
assert all("ali" in r[3] for r in rows)
regex = {
**base,
"filters": [{"column": "user", "op": "~", "value": "^a.*"}],
}
rv = client.post(
"/api/query", data=json.dumps(regex), content_type="application/json"
)
rows = rv.get_json()["rows"]
assert all(r[3].startswith("a") for r in rows)
assert len(rows) == 2
not_empty = {**base, "filters": [{"column": "user", "op": "!empty"}]}
rv = client.post(
"/api/query", data=json.dumps(not_empty), content_type="application/json
"
)
assert len(rv.get_json()["rows"]) == 4
def _make_payload() -> dict[str, object]:
return {
"table": "events",
"start": "2024-01-01 00:00:00","start": "2024-01-01 00:00:00",
"end": "2024-01-02 00:00:00",
"order_by": "timestamp",
"order_dir": "ASC",
"limit": 10,
"columns": ["timestamp", "event", "value", "user"],
"filters": [],
}
def test_database_types(tmp_path: Path) -> None:
csv_file = tmp_path / "events.csv"
csv_file.write_text(Path("scubaduck/sample.csv").read_text())
sqlite_file = tmp_path / "events.sqlite"
import sqlite3
conn = sqlite3.connect(sqlite_file)
conn.execute(
"CREATE TABLE events (timestamp TEXT, event TEXT, value INTEGER, user TE
XT)"
)
with open(csv_file) as f:
next(f)
for line in f:
ts, ev, val, user = line.strip().split(",")
conn.execute(
"INSERT INTO events VALUES (?, ?, ?, ?)", (ts, ev, int(val), use
r)
)
conn.commit()
conn.close() # pyright: ignore[reportUnknownMemberType, reportAttributeAcce
ssIssue]
duckdb_file = tmp_path / "events.duckdb"
con = duckdb.connect(duckdb_file)
con.execute(
f"CREATE TABLE events AS SELECT * FROM read_csv_auto('{csv_file.as_posix
()}')"
)
con.close() # pyright: ignore[reportUnknownMemberType, reportAttributeAcces
sIssue]
for db in (csv_file, sqlite_file, duckdb_file):
app = server.create_app(db)
client = app.test_client()
payload = _make_payload()
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/js
on"
)
rows = rv.get_json()["rows"]
assert len(rows) == 3
def test_sqlite_longvarchar(tmp_path: Path) -> None:
sqlite_file = tmp_path / "events.sqlite"
import sqlite3
conn = sqlite3.connect(sqlite_file)
conn.execute(
"CREATE TABLE events (timestamp TEXT, url LONGVARCHAR, title VARCHAR(10)
)"
)
conn.execute(
"INSERT INTO events VALUES ('2024-01-01 00:00:00', 'https://a.com', 'Hom
e')"
)
conn.commit()
conn.close() # pyright: ignore[reportUnknownMemberType, reportAttributeAcce
ssIssue]
app = server.create_app(sqlite_file)
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-01 01:00:00",
"order_by": "timestamp",
"columns": ["timestamp", "url", "title"],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert data["rows"][0][1] == "https://a.com"
def test_sqlite_bigint(tmp_path: Path) -> None:
sqlite_file = tmp_path / "big.sqlite"
import sqlite3
conn = sqlite3.connect(sqlite_file)
conn.execute("CREATE TABLE events (timestamp TEXT, value INTEGER)")
big_value = 13385262862605259
conn.execute(
"INSERT INTO events VALUES ('2024-01-01 00:00:00', ?)",
(big_value,),
)
conn.commit()
conn.close() # pyright: ignore[reportUnknownMemberType, reportAttributeAcce
ssIssue]
app = server.create_app(sqlite_file)
client = app.test_client()
payload = {
"table": "events",
"order_by": "timestamp",
"columns": ["timestamp", "value"],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert data["rows"][0][1] == big_value
def test_sqlite_boolean_aggregation(tmp_path: Path) -> None:
sqlite_file = tmp_path / "bool.sqlite"
import sqlite3
conn = sqlite3.connect(sqlite_file)
conn.execute("CREATE TABLE events (timestamp TEXT, flag BOOLEAN)")
conn.execute("INSERT INTO events VALUES ('2024-01-01 00:00:00', 1)")
conn.execute("INSERT INTO events VALUES ('2024-01-01 00:30:00', 0)")
conn.commit()
conn.close() # pyright: ignore[reportUnknownMemberType, reportAttributeAcce
ssIssue]
app = server.create_app(sqlite_file)
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-02 00:00:00",
"graph_type": "table",
"aggregate": "Avg",
"columns": ["flag"],
"show_hits": True,
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert data["rows"][0][0] == 2
assert data["rows"][0][1] == 0.5
def test_integer_time_column(tmp_path: Path) -> None:
csv_file = tmp_path / "events.csv"
csv_file.write_text("created,event\n1704067200,login\n1704070800,logout\n")
app = server.create_app(csv_file)
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-01 01:00:00",
"order_by": "created",
"columns": ["created", "event"],
"time_column": "created",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert len(data["rows"]) == 2"time_unit": "ms",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert len(data["rows"]) == 2
def test_timeseries_default_xaxis_uses_time_column(tmp_path: Path) -> None:
csv_file = tmp_path / "events.csv"
csv_file.write_text("created,event\n1704067200000,login\n1704070800000,logou
t\n")
app = server.create_app(csv_file)
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-01 01:00:00",
"graph_type": "timeseries",
"granularity": "1 hour",
"columns": ["event"],
"aggregate": "Count",
"time_column": "created",
"time_unit": "ms",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert len(data["rows"]) == 2
def test_integer_time_unit_us_default_start_end(tmp_path: Path) -> None:
csv_file = tmp_path / "events.csv"
csv_file.write_text(
"created,event\n1704067200000000,login\n1704070800000000,logout\n"
)
app = server.create_app(csv_file)
client = app.test_client()
payload = {
"table": "events",
"order_by": "created",
"columns": ["created", "event"],
"time_column": "created",
"time_unit": "us",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert data["start"] == "2024-01-01 00:00:00"
assert data["end"] == "2024-01-01 01:00:00"
assert len(data["rows"]) == 2
def test_sqlite_integer_time_unit_us(tmp_path: Path) -> None:
sqlite_file = tmp_path / "events.sqlite"
import sqlite3
conn = sqlite3.connect(sqlite_file)
conn.execute("CREATE TABLE visits (visit_time INTEGER, event TEXT)")
big_ts = 13384551652000000
conn.execute("INSERT INTO visits VALUES (?, ?)", (big_ts, "foo"))
conn.commit()
conn.close() # pyright: ignore[reportUnknownMemberType, reportAttributeAcce
ssIssue]
app = server.create_app(sqlite_file)
client = app.test_client()
payload = {
"table": "visits",
"start": "2394-02-20 00:00:00",
"end": "2394-02-21 00:00:00",
"order_by": "visit_time",
"columns": ["visit_time", "event"],
"time_column": "visit_time",
"time_unit": "us",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert len(data["rows"]) == 1
def test_envvar_db(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
csv_file = tmp_path / "custom.csv"
csv_file.write_text("timestamp,event,value,user\n2024-01-01 00:00:00,login,5
,bob\n")
monkeypatch.setenv("SCUBADUCK_DB", str(csv_file))
app = server.create_app()
client = app.test_client()
payload = _make_payload()
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
rows = rv.get_json()["rows"]
assert len(rows) == 1
def test_envvar_db_missing(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> N
one:
missing = tmp_path / "missing.sqlite"
monkeypatch.setenv("SCUBADUCK_DB", str(missing))
with pytest.raises(FileNotFoundError):
server.create_app()
def test_group_by_table() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "table",
"order_by": "user",
"limit": 10,
"columns": ["value"],
...
def test_query_error_returns_sql_and_traceback() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"columns": ["event"],
"group_by": ["user"],
"aggregate": "avg",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 400
assert "error" in data
def test_table_unknown_column_error() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "table",
"order_by": "timestamp",
"limit": 100,
"columns": ["user", "Hits", "value"],
"group_by": ["user"],
"aggregate": "Count",
"show_hits": True,
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 400
assert "Unknown column" in data["error"]
def test_samples_view_rejects_group_by() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"graph_type": "samples",
"group_by": ["user"],
"columns": ["timestamp"],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 400
assert "only valid" in data["error"]
def test_table_avg_with_timestamp() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "table",
"order_by": "user",
"limit": 100,
"columns": ["user", "timestamp", "value"],
"group_by": ["user"],
"aggregate": "Avg",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert "error" not in data
rows = data["rows"]
assert rows[0][0] == "alice"
from dateutil import parser
ts = parser.parse(rows[0][1]).replace(tzinfo=None)
assert ts == parser.parse("2024-01-01 12:00:00")
def test_timeseries_basic() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "timeseries",
"limit": 100,
"group_by": ["user"],
"aggregate": "Count",
"columns": ["value"],
"x_axis": "timestamp",
"granularity": "1 day",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert len(data["rows"]) == 4
def test_timeseries_orders_by_xaxis() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "timeseries",
"limit": 100,
"columns": ["value"],
"x_axis": "timestamp",
"granularity": "1 day",rows = data["rows"]
from dateutil import parser
timestamps = [parser.parse(r[0]).replace(tzinfo=None) for r in rows]
assert timestamps == sorted(timestamps)
def test_timeseries_count_no_columns() -> None:
app = server.app
client = app.test_client()
payload: dict[str, Any] = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "timeseries",
"granularity": "1 day",
"columns": [],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
rows = data["rows"]
assert len(rows) == 2
assert rows[0][1] == 2
assert rows[1][1] == 2
def test_timeseries_limit_applies_to_series() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "timeseries",
"limit": 1,
"order_by": "user",
"group_by": ["user"],
"aggregate": "Count",
"columns": ["value"],
"x_axis": "timestamp",
"granularity": "1 day",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert len(data["rows"]) == 2
assert all(r[1] == "alice" for r in data["rows"])
def test_timeseries_auto_and_fine_buckets() -> None:
app = server.app
client = app.test_client()
def run(gran: str) -> None:
payload = {
"start": "2024-01-01 00:00:00",
"end": "2024-01-02 03:00:00",
"graph_type": "timeseries",
"columns": ["value"],
"x_axis": "timestamp",
"granularity": gran,
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/js
on"
)
data = rv.get_json()
assert rv.status_code == 200
from dateutil import parser
start = parser.parse(cast(str, payload["start"])).replace(tzinfo=None)
buckets = [
parser.parse(cast(str, r[0])).replace(tzinfo=None) for r in data["ro
ws"]
]
assert buckets[0] == start
if len(buckets) > 1:
step = (buckets[1] - buckets[0]).total_seconds()
assert step % data["bucket_size"] == 0
assert any(r[1] != 0 for r in data["rows"])
run("Auto")
run("Fine")
def test_timeseries_string_column_error() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "timeseries",
"limit": 7,
"columns": ["timestamp", "event", "value", "user"],
"x_axis": "timestamp",
"granularity": "1 hour",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
assert rv.status_code == 200
def test_derived_column_basic() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"limit": 10,
"columns": ["timestamp"],
"derived_columns": {"val2": "value * 2"},
"filters": [],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert data["rows"][0][1] == 20
def test_timeseries_derived_column() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "timeseries",
"granularity": "1 hour",
"limit": 7,
"columns": ["value"],
"derived_columns": {"derived_1": "value * 2"},
"aggregate": "Avg",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
rows = data["rows"]
assert all(r[2] == r[1] * 2 for r in rows)
def test_default_start_end_returned() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"order_by": "timestamp",
"limit": 5,
"columns": ["timestamp"],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert data["start"] == "2024-01-01 00:00:00"
assert data["end"] == "2024-01-02 03:00:00"
def test_time_column_none_no_time_filter() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"columns": ["timestamp", "event"],
"time_column": "",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert "start" not in data and "end" not in data
assert len(data["rows"]) == 4root@eb913c2329bf:/workspace/scubaduck# sed -n '960,1280p' tests/test_server.pyI see some tests related to quoting and others in test_multi_table_web.py. Let's dig!
root@eb913c2329bf:/workspace/scubaduck# grep -n "extra" -n tests/test_server.pyI'll look for any tests referring to "extra" or possibly quoting.
root@eb913c2329bf:/workspace/scubaduck# grep -n "desc" -r testsI'm considering proper quoting for SQL columns to prevent issues with the "desc" keyword.
I'll quote columns using f'"{col}"', but skip for aggregator functions and expressions.
sec = _granularity_seconds(params.granularity, params.start, params.end)
x_axis = params.x_axis or params.time_column
if x_axis is None:
raise ValueError("x_axis required for timeseries")
xexpr = _time_expr(x_axis, column_types, params.time_unit)
if params.start:
bucket_expr = (
f"TIMESTAMP '{params.start}' + INTERVAL '{sec} second' * "
f"CAST(floor((epoch({xexpr}) - epoch(TIMESTAMP '{params.start}')
)/{sec}) AS BIGINT)"
)
else:
bucket_expr = (
f"TIMESTAMP 'epoch' + INTERVAL '{sec} second' * "
f"CAST(floor(epoch({xexpr})/{sec}) AS BIGINT)"
)
select_parts.append(f"{bucket_expr} AS bucket")
group_cols = ["bucket"] + group_cols
has_agg = bool(group_cols) or params.aggregate is not None
if has_agg:
select_cols = (
group_cols[1:] if params.graph_type == "timeseries" else group_cols
)
select_parts.extend(select_cols)
agg = (params.aggregate or "count").lower()
def agg_expr(col: str) -> str:
expr = col
ctype = column_types.get(col, "").upper() if column_types else ""
if "BOOL" in ctype:
expr = f"CAST({col} AS BIGINT)"
if agg.startswith("p"):
quant = float(agg[1:]) / 100
return f"quantile({expr}, {quant})"
if agg == "count distinct":
return f"count(DISTINCT {expr})"
if agg == "avg" and column_types is not None:
if "TIMESTAMP" in ctype or "DATE" in ctype or "TIME" in ctype:
return (
"TIMESTAMP 'epoch' + INTERVAL '1 second' * "
f"CAST(avg(epoch({col})) AS BIGINT)"
)
return f"{agg}({expr})"
if agg == "count":
select_parts.append("count(*) AS Count")
else:
for col in params.columns:
if col in group_cols:
continue
select_parts.append(f"{agg_expr(col)} AS {col}")
if params.show_hits:
select_parts.insert(len(group_cols), "count(*) AS Hits")
else:
select_parts.extend(params.columns)
if has_agg and params.derived_columns:
inner_params = replace(
params,
derived_columns={},
order_by=None,
limit=None,
)
inner_sql = build_query(inner_params, column_types)
outer_select = ["t.*"] + [
f"{expr} AS {name}" for name, expr in params.derived_columns.items()
]
indented_inner = "\n".join(" " + line for line in inner_sql.splitline
s())
lines = [
f"SELECT {', '.join(outer_select)}",
"FROM (",
indented_inner,
") t",
]
if params.order_by:
lines.append(f"ORDER BY {params.order_by} {params.order_dir}")
elif params.graph_type == "timeseries":
lines.append("ORDER BY bucket")
if params.limit is not None:
lines.append(f"LIMIT {params.limit}")
return "\n".join(lines)
for name, expr in params.derived_columns.items():
select_parts.append(f"{expr} AS {name}")
select_clause = ", ".join(select_parts) if select_parts else "*"
lines = [f"SELECT {select_clause}", f'FROM "{params.table}"']
where_parts: list[str] = []
if params.time_column:
time_expr = _time_expr(params.time_column, column_types, params.time_uni
t)
else:
time_expr = None
if time_expr and params.start:
where_parts.append(f"{time_expr} >= '{params.start}'")
if time_expr and params.end:
where_parts.append(f"{time_expr} <= '{params.end}'")
for f in params.filters:
op = f.op
if op in {"empty", "!empty"}:
val = "''"
else:
if f.value is None:
continue
if isinstance(f.value, list):
if not f.value:
continue
if op == "=":
vals = " OR ".join(f"{f.column} = '{v}'" for v in f.value)
where_parts.append(f"({vals})")
continue
val = f"'{f.value}'" if isinstance(f.value, str) else str(f.value)
if op == "contains":
where_parts.append(f"{f.column} ILIKE '%' || {val} || '%'")
elif op == "!contains":
where_parts.append(f"{f.column} NOT ILIKE '%' || {val} || '%'")
elif op == "empty":
where_parts.append(f"{f.column} = {val}")
elif op == "!empty":
where_parts.append(f"{f.column} != {val}")
else:
where_parts.append(f"{f.column} {op} {val}")
if where_parts:
lines.append("WHERE " + " AND ".join(where_parts))
if group_cols:
lines.append("GROUP BY " + ", ".join(group_cols))
if params.order_by:
lines.append(f"ORDER BY {params.order_by} {params.order_dir}")
elif params.graph_type == "timeseries":
lines.append("ORDER BY bucket")
if params.limit is not None:
lines.append(f"LIMIT {params.limit}")
return "\n".join(lines)
def create_app(db_file: str | Path | None = None) -> Flask:
app = Flask(__name__, static_folder="static")
if db_file is None:
env_db = os.environ.get("SCUBADUCK_DB")
if env_db:
db_file = env_db
if isinstance(db_file, str) and db_file.upper() == "TEST":
con = _create_test_database()
else:
db_path = Path(db_file or Path(__file__).with_name("sample.csv")).resolv
e()
con = _load_database(db_path)
tables = [r[0] for r in con.execute("SHOW TABLES").fetchall()]
if not tables:
raise ValueError("No tables found in database")
default_table = tables[0]
columns_cache: Dict[str, Dict[str, str]] = {}
def get_columns(table: str) -> Dict[str, str]:
if table not in columns_cache:
rows = con.execute(f'PRAGMA table_info("{table}")').fetchall()
if not rows:
raise ValueError(f"Unknown table: {table}")
columns_cache[table] = {r[1]: r[2] for r in rows}
return columns_cache[table]
sample_cache: Dict[Tuple[str, str, str], Tuple[List[str], float]] = {}
CACHE_TTL = 60.0
CACHE_LIMIT = 200
@app.route("/")
def index() -> Any: # pyright: ignore[reportUnusedFunction]
assert app.static_folder is not None
return send_from_directory(app.static_folder, "index.html")
@app.route("/js/<path:filename>")
def js(filename: str) -> Any: # pyright: ignore[reportUnusedFunction]
assert app.static_folder is not None
folder = Path(app.static_folder) / "js"
return send_from_directory(folder, filename)
@app.route("/api/tables")
def tables_endpoint() -> Any: # pyright: ignore[reportUnusedFunction]
return jsonify(tables)
@app.route("/api/columns")
def columns() -> Any: # pyright: ignore[reportUnusedFunction]
table = request.args.get("table", default_table)
rows = con.execute(f'PRAGMA table_info("{table}")').fetchall()
return jsonify([{"name": r[1], "type": r[2]} for r in rows])
def _cache_get(key: Tuple[str, str, str]) -> List[str] | None:
item = sample_cache.get(key)
if item is None:
return None
vals, ts = item
if time.time() - ts > CACHE_TTL:
del sample_cache[key]
return None
sample_cache[key] = (vals, time.time())
return vals
def _cache_set(key: Tuple[str, str, str], vals: List[str]) -> None:
sample_cache[key] = (vals, time.time())
if len(sample_cache) > CACHE_LIMIT:
oldest = min(sample_cache.items(), key=lambda kv: kv[1][1])[0]
del sample_cache[oldest]root@eb913c2329bf:/workspace/scubaduck# grep -n "sample_values" -n scubaduck/ser
ver.py
402: def sample_values() -> Any: # pyright: ignore[reportUnusedFunction]root@eb913c2329bf:/workspace/scubaduck# sed -n '406,440p' scubaduck/server.py
column_types = get_columns(table)
if not column or column not in column_types:
return jsonify([])
ctype = column_types[column].upper()
if "CHAR" not in ctype and "STRING" not in ctype and "VARCHAR" not in ct
ype:
return jsonify([])
key = (table, column, substr)
cached = _cache_get(key)
if cached is not None:
return jsonify(cached)
rows = con.execute(
f"SELECT DISTINCT {column} FROM \"{table}\" WHERE CAST({column} AS V
ARCHAR) ILIKE '%' || ? || '%' LIMIT 20",
[substr],
).fetchall()
values = [r[0] for r in rows]
_cache_set(key, values)
return jsonify(values)
@app.route("/api/query", methods=["POST"])
def query() -> Any: # pyright: ignore[reportUnusedFunction]
payload = request.get_json(force=True)
try:
start = parse_time(payload.get("start"))
end = parse_time(payload.get("end"))
except Exception as exc:
return jsonify({"error": str(exc)}), 400
params = QueryParams(
start=start,
end=end,
order_by=payload.get("order_by"),
order_dir=payload.get("order_dir", "ASC"),
limit=payload.get("limit"),
columns=payload.get("columns", []),
derived_columns=payload.get("derived_columns", {}),I’ll modify sample_values to include quoting and add tests for verification. Let's explore test files.
eb.py
import threading
from collections.abc import Iterator
from typing import Any
import pytest
from werkzeug.serving import make_server
from scubaduck.server import create_app
from tests.test_web import select_value
@pytest.fixture()
def multi_table_server_url() -> Iterator[str]:
app = create_app("TEST")
httpd = make_server("127.0.0.1", 0, app)
port = httpd.server_port
thread = threading.Thread(target=httpd.serve_forever)
thread.start()
try:
yield f"http://127.0.0.1:{port}"
finally:
httpd.shutdown()
thread.join()
def test_table_param_updates_on_dive(page: Any, multi_table_server_url: str) ->
None:
page.goto(multi_table_server_url + "?table=events")
page.wait_for_selector("#table option", state="attached")
select_value(page, "#table", "extra")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
table_param = page.evaluate(
"new URLSearchParams(window.location.search).get('table')"
)
assert table_param == "extra"I need to scroll a bit more to get all the necessary information.
root@eb913c2329bf:/workspace/scubaduck# sed -n '160,320p' tests/test_multi_table
_web.py{"sel": selector, "val": value},
)
def run_query(
page: Any,
url: str,
*,
start: str | None = None,
end: str | None = None,
order_by: str | None = None,
order_dir: str | None = "ASC",
limit: int | None = None,
group_by: list[str] | None = None,
aggregate: str | None = None,
) -> dict[str, Any]:
page.goto(url)
page.wait_for_selector("#order_by option", state="attached")
page.wait_for_selector("#order_dir", state="attached")
page.wait_for_function("window.lastResults !== undefined")
if start is not None:
page.fill("#start", start)
if end is not None:
page.fill("#end", end)
if order_by is not None:
select_value(page, "#order_by", order_by)
if order_dir is not None and order_dir == "DESC":
page.click("#order_dir")
if limit is not None:
page.fill("#limit", str(limit))
if group_by is not None:
select_value(page, "#graph_type", "table")
page.evaluate(
"g => { groupBy.chips = g; groupBy.renderChips(); }",
group_by,
)
if aggregate is not None:
select_value(page, "#graph_type", "table")
select_value(page, "#aggregate", aggregate)
if page.input_value("#graph_type") != "samples":
page.click("text=Columns")
page.wait_for_selector("#column_groups input", state="attached")
if not page.is_checked("#column_groups input[value='value']"):
page.check("#column_groups input[value='value']")
order_col = order_by or page.input_value("#order_by")
if order_col and not page.is_checked(
f"#column_groups input[value='{order_col}']"
):
if page.query_selector(f"#column_groups input[value='{order_col}']")
:
page.check(f"#column_groups input[value='{order_col}']")
page.click("text=View Settings")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
return page.evaluate("window.lastResults")
def test_range_filters(page: Any, server_url: str) -> None:
data = run_query(
page,
server_url,
start="2024-01-02 00:00:00",
end="2024-01-02 04:00:00",
order_by="user",
limit=100,
)
assert len(data["rows"]) == 2
from dateutil import parser
timestamps = [parser.parse(row[0]).replace(tzinfo=None) for row in data["row
s"]]
assert timestamps == [
parser.parse("2024-01-02 00:00:00"),
parser.parse("2024-01-02 03:00:00"),
]
def test_order_by(page: Any, server_url: str) -> None:
data = run_query(
page,
server_url,
start="2024-01-01 00:00:00",
end="2024-01-03 00:00:00",
order_by="value",
order_dir="DESC",
limit=100,
)
values = [row[2] for row in data["rows"]]
assert values == sorted(values, reverse=True)
def test_limit(page: Any, server_url: str) -> None:
data = run_query(
page,
server_url,
start="2024-01-01 00:00:00",
end="2024-01-03 00:00:00",
order_by="user",
limit=2,
)
assert len(data["rows"]) == 2
def test_time_column_dropdown(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#time_column option", state="attached")
options = page.locator("#time_column option").all_inner_texts()
assert "(none)" in options
assert "timestamp" in options
assert "value" in options
assert page.input_value("#time_column") == "timestamp"
def test_time_column_none_hides_range(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#time_column option", state="attached")
select_value(page, "#time_column", "")
assert page.is_hidden("#start")
assert page.is_hidden("#end")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
data = page.evaluate("window.lastResults")
assert len(data["rows"]) == 4
assert "start" not in data and "end" not in data
def test_time_unit_dropdown(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#time_unit", state="attached")
opts = page.locator("#time_unit option").all_inner_texts()
assert "ms" in opts
assert page.input_value("#time_unit") == "s"
def test_table_selector_dropdown(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#table option", state="attached")
disp = page.query_selector("#table + .dropdown-display")
assert disp
assert (
page.evaluate("getComputedStyle(document.querySelector('#table')).displa
y")
== "none"
)
assert page.query_selector("#table + .dropdown-display + .dropdown-menu inpu
t")
def test_x_axis_default_entry(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#graph_type", state="attached")
select_value(page, "#graph_type", "timeseries")
page.wait_for_selector("#x_axis option", state="attached")
options = page.locator("#x_axis option").all_inner_texts()48: if page.input_value("#graph_type") != "samples":
215: assert "sample.csv" in header
271: select_value(page, "#graph_type", "samples")
286: select_value(page, "#graph_type", "samples")I'll focus on implementing quoting now, and review the README to confirm it's missing.
./scubaduck/static/index.html:1276: label + (sortState.dir === 'desc' ? '
\u25BC' : ' \u25B2');
./scubaduck/static/index.html:1347: sortState.dir = 'desc';
./scubaduck/static/index.html:1348: } else if (sortState.dir === 'desc') {
./scubaduck/static/index.html:1360: if (sortState.dir === 'desc') return va
< vb ? 1 : -1;
./scubaduck/server.py:84: con.execute('CREATE TABLE extra (ts TEXT, "desc" TE
XT, num INTEGER)')
./.venv/lib/python3.12/site-packages/click/decorators.py:101: key: str, *, do
c_description: str | None = None
./.venv/lib/python3.12/site-packages/click/decorators.py:108: :param doc_desc
ription: Description of the object being passed,
./.venv/lib/python3.12/site-packages/click/decorators.py:123: if doc_descript
ion is None:
./.venv/lib/python3.12/site-packages/click/decorators.py:124: doc_descrip
tion = f"the {key!r} key from :attr:`click.Context.meta`"
./.venv/lib/python3.12/site-packages/click/decorators.py:127: f"Decorator
that passes {doc_description} as the first argument"
./.venv/lib/python3.12/site-packages/click/decorators.py:191: name as des
cribed above.
./.venv/lib/python3.12/site-packages/click/decorators.py:334: :class:`Paramet
er` for descriptions of parameters.
./.venv/lib/python3.12/site-packages/click/decorators.py:362: :class:`Paramet
er` for descriptions of parameters.
./.venv/lib/python3.12/site-packages/click/core.py:183: is
the most descriptive name for the script or
./.venv/lib/python3.12/site-packages/click/core.py:295: #: the descriptiv
e information name
./.venv/lib/python3.12/site-packages/click/core.py:1167:
is the most descriptive name for the script or
./.venv/lib/python3.12/site-packages/click/core.py:2050: semantics descri
bed above. The old name is deprecated and will
./.venv/lib/python3.12/site-packages/click/core.py:2509: :param attrs: Other
command arguments described in :class:`Parameter`.
./.venv/lib/python3.12/site-packages/click/core.py:2891: range_str =
self.type._describe_range()
./.venv/lib/python3.12/site-packages/click/types.py:51: #: the descriptive na
me of this type
./.venv/lib/python3.12/site-packages/click/types.py:116: descriptive mess
age.
./.venv/lib/python3.12/site-packages/click/types.py:539: valu
e=rv, range=self._describe_range()
./.venv/lib/python3.12/site-packages/click/types.py:557: def _describe_range(
self) -> str:
./.venv/lib/python3.12/site-packages/click/types.py:573: return f"<{type(
self).__name__} {self._describe_range()}{clamp}>"
./.venv/lib/python3.12/site-packages/click/shell_completion.py:132: local -a
completions_with_descriptions
./.venv/lib/python3.12/site-packages/click/shell_completion.py:139: for type
key descr in ${response}; do
./.venv/lib/python3.12/site-packages/click/shell_completion.py:141: i
f [[ "$descr" == "_" ]]; then
./.venv/lib/python3.12/site-packages/click/shell_completion.py:144:
completions_with_descriptions+=("$key":"$descr")
./.venv/lib/python3.12/site-packages/click/shell_completion.py:153: if [ -n "
$completions_with_descriptions" ]; then
./.venv/lib/python3.12/site-packages/click/shell_completion.py:154: _desc
ribe -V unsorted completions_with_descriptions -U
grep: ./.venv/lib/python3.12/site-packages/greenlet/_greenlet.cpython-312-x86_64
-linux-gnu.so: binary file matches
./.venv/lib/python3.12/site-packages/greenlet/TPythonState.cpp:351: // descri
bed below, our pointers may not even be valid anymore. As
./.venv/lib/python3.12/site-packages/greenlet/PyGreenlet.cpp:402: "propagatio
n rules apply, as described for `switch`. Note that calling "
./.venv/lib/python3.12/site-packages/greenlet/TUserGreenlet.cpp:246: * _
_getattribute__ or __getattr__, or makes ``run`` a descriptor;
./.venv/lib/python3.12/site-packages/greenlet/tests/test_greenlet.py:1254:
# the ``inner_bootstrap()`` C frame and its descendents hanging around,
grep: ./.venv/lib/python3.12/site-packages/greenlet/tests/_test_extension_cpp.cp
ython-312-x86_64-linux-gnu.so: binary file matches
./.venv/lib/python3.12/site-packages/slugify/__main__.py:11: parser = argpars
e.ArgumentParser(description="Slug string")
./.venv/lib/python3.12/site-packages/slugify/__main__.py:13: input_group = pa
rser.add_argument_group(description="Input")
./.venv/lib/python3.12/site-packages/slugify/__init__.py:6:from .__version__ imp
ort __description__
./.venv/lib/python3.12/site-packages/slugify/__version__.py:4:__description__ =
'A Python slugify application that also handles Unicode'
./.venv/lib/python3.12/site-packages/certifi-2025.4.26.dist-info/METADATA:30:Dyn
amic: description
./.venv/lib/python3.12/site-packages/duckdb-stubs/__init__.pyi:256: def desc(
self) -> "Expression": ...
./.venv/lib/python3.12/site-packages/duckdb-stubs/__init__.pyi:279: def descr
iption(self) -> Optional[List[Any]]: ...
./.venv/lib/python3.12/site-packages/duckdb-stubs/__init__.pyi:419: def descr
ibe(self) -> DuckDBPyRelation: ...
./.venv/lib/python3.12/site-packages/duckdb-stubs/__init__.pyi:536: def descr
iption(self) -> List[Any]: ...
./.venv/lib/python3.12/site-packages/duckdb-stubs/__init__.pyi:706:def descripti
on(*, connection: DuckDBPyConnection = ...) -> Optional[List[Any]]: ...
./.venv/lib/python3.12/site-packages/pytest_playwright-0.7.0.dist-info/METADATA:
145: except as required for reasonable and customary use in describ
ing the
./.venv/lib/python3.12/site-packages/pytest_playwright-0.7.0.dist-info/METADATA:
190: file or class name and description of purpose be included on t
he
./.venv/lib/python3.12/site-packages/pytest_playwright-0.7.0.dist-info/LICENSE:1
40: except as required for reasonable and customary use in describing the
./.venv/lib/python3.12/site-packages/pytest_playwright-0.7.0.dist-info/LICENSE:1
85: file or class name and description of purpose be included on the
./.venv/lib/python3.12/site-packages/charset_normalizer/utils.py:30: desc
ription: str = unicodedata.name(character)
./.venv/lib/python3.12/site-packages/charset_normalizer/utils.py:34: "WIT
H GRAVE" in description
./.venv/lib/python3.12/site-packages/charset_normalizer/utils.py:35: or "
WITH ACUTE" in description
./.venv/lib/python3.12/site-packages/charset_normalizer/utils.py:36: or "
WITH CEDILLA" in description
./.venv/lib/python3.12/site-packages/charset_normalizer/utils.py:37: or "
WITH DIAERESIS" in description
./.venv/lib/python3.12/site-packages/charset_normalizer/utils.py:38: or "
WITH CIRCUMFLEX" in description
./.venv/lib/python3.12/site-packages/charset_normalizer/utils.py:39: or "
WITH TILDE" in description
./.venv/lib/python3.12/site-packages/charset_normalizer/utils.py:40: or "
WITH MACRON" in description
./.venv/lib/python3.12/site-packages/charset_normalizer/utils.py:41: or "
WITH RING ABOVE" in description
./.venv/lib/python3.12/site-packages/charset_normalizer/utils.py:73: desc
ription: str = unicodedata.name(character)
./.venv/lib/python3.12/site-packages/charset_normalizer/utils.py:76: return "
LATIN" in description
./.venv/lib/python3.12/site-packages/charset_normalizer/cli/__main__.py:116:
description="The Real First Universal Charset Detector. "
./.venv/lib/python3.12/site-packages/requests-2.32.3.dist-info/LICENSE:141:
except as required for reasonable and customary use in describing the
./.venv/lib/python3.12/site-packages/xdist/report.py:15: :returns: detailed m
essage describing the difference between the given
grep: ./.venv/lib/python3.12/site-packages/markupsafe/_speedups.cpython-312-x86_
64-linux-gnu.so: binary file matches
./.venv/lib/python3.12/site-packages/six.py:101: # removing this desc
riptor.
./.venv/lib/python3.12/site-packages/jinja2/utils.py:165: """Returns a file d
escriptor for the filename if that file exists,
./.venv/lib/python3.12/site-packages/jinja2/filters.py:401: :param reverse: S
ort descending instead of ascending.
./.venv/lib/python3.12/site-packages/jinja2/filters.py:1425: # This avoid
s executing properties/descriptors, but misses __getattr__
./.venv/lib/python3.12/site-packages/jinja2/filters.py:1429: # This finds
dynamic attrs, and we know it's not a descriptor at this point.
./.venv/lib/python3.12/site-packages/jinja2/lexer.py:165:def _describe_token_typ
e(token_type: str) -> str:
./.venv/lib/python3.12/site-packages/jinja2/lexer.py:185:def describe_token(toke
n: "Token") -> str:
./.venv/lib/python3.12/site-packages/jinja2/lexer.py:186: """Returns a descri
ption of the token."""
./.venv/lib/python3.12/site-packages/jinja2/lexer.py:190: return _describe_to
ken_type(token.type)
./.venv/lib/python3.12/site-packages/jinja2/lexer.py:193:def describe_token_expr
(expr: str) -> str:
./.venv/lib/python3.12/site-packages/jinja2/lexer.py:194: """Like `describe_t
oken` but for token expressions."""
./.venv/lib/python3.12/site-packages/jinja2/lexer.py:203: return _describe_to
ken_type(type)
./.venv/lib/python3.12/site-packages/jinja2/lexer.py:275: return describe
_token(self)
./.venv/lib/python3.12/site-packages/jinja2/lexer.py:408: expr = desc
ribe_token_expr(expr)
./.venv/lib/python3.12/site-packages/jinja2/lexer.py:419: f"expec
ted token {expr!r}, got {describe_token(self.current)!r}",
./.venv/lib/python3.12/site-packages/jinja2/parser.py:9:from .lexer import descr
ibe_token
./.venv/lib/python3.12/site-packages/jinja2/parser.py:10:from .lexer import desc
ribe_token_expr
./.venv/lib/python3.12/site-packages/jinja2/parser.py:98: expected.up
date(map(describe_token_expr, exprs))
./.venv/lib/python3.12/site-packages/jinja2/parser.py:101: map(re
pr, map(describe_token_expr, end_token_stack[-1]))
./.venv/lib/python3.12/site-packages/jinja2/parser.py:682: self.fail(
f"unexpected {describe_token(token)!r}", token.lineno)
./.venv/lib/python3.12/site-packages/jinja2/parser.py:749: f"
got {describe_token(self.stream.current)!r}"
./.venv/lib/python3.12/site-packages/greenlet-3.2.2.dist-info/METADATA:44:Dynami
c: description
./.venv/lib/python3.12/site-packages/greenlet-3.2.2.dist-info/METADATA:45:Dynami
c: description-content-type
./.venv/lib/python3.12/site-packages/urllib3/util/wait.py:22:# altogether, for h
igh-numbered file descriptors. The point of poll() is to fix
./.venv/lib/python3.12/site-packages/urllib3/response.py:1113: "a
round has no file descriptor"
./.venv/lib/python3.12/site-packages/text_unidecode-1.3.dist-info/metadata.json:
1:{"classifiers": ["Development Status :: 5 - Production/Stable", "Intended Audi
ence :: Developers", "License :: OSI Approved :: Artistic License", "License ::
OSI Approved :: GNU General Public License (GPL)", "License :: OSI Approved :: G
NU General Public License v2 or later (GPLv2+)", "Programming Language :: Python
", "Programming Language :: Python :: 2", "Programming Language :: Python :: 2.7
", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.4
", "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3
.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python ::
Implementation :: CPython", "Programming Language :: Python :: Implementation :
: PyPy", "Topic :: Software Development :: Libraries :: Python Modules", "Topic
:: Text Processing :: Linguistic"], "extensions": {"python.details": {"contacts"
: [{"email": "kmike84@gmail.com", "name": "Mikhail Korobov", "role": "author"}],
"document_names": {"description": "DESCRIPTION.rst", "license": "LICENSE.txt"},
"project_urls": {"Home": "https://github.com/kmike/text-unidecode/"}}}, "genera
tor": "bdist_wheel (0.29.0)", "license": "Artistic License", "metadata_version":
"2.0", "name": "text-unidecode", "summary": "The most basic Text::Unidecode por
t", "version": "1.3"}
./.venv/lib/python3.12/site-packages/nodeenv.py:479: cmd_desc = ' '.join(cmd_
parts)
./.venv/lib/python3.12/site-packages/nodeenv.py:480: logger.debug(" ** Runnin
g command %s" % cmd_desc)
./.venv/lib/python3.12/site-packages/nodeenv.py:503: logger.error("Error
%s while executing command %s" % (e, cmd_desc))
./.venv/lib/python3.12/site-packages/nodeenv.py:529: % (cmd
_desc, proc.returncode))
./.venv/lib/python3.12/site-packages/pyright/_utils.py:20: 'description': 'Ca
che directory created by Pyright Python to store downloads of the NPM package',
./.venv/lib/python3.12/site-packages/pyright/dist/package.json:4: "descriptio
n": "Type checker for the Python language",bash: .venv/bin/activate: No such file or directory
root@eb913c2329bf:/workspace# grep -R "create_app(\"TEST\")" -n tests
grep: tests: No such file or directoryLooks like I was in the wrong folder. Time to switch to the repo directory!
root@eb913c2329bf:/workspace# cd scubaduck
root@eb913c2329bf:/workspace/scubaduck# ls
AGENTS.md codex_setup.sh scubaduck tests
README.md pyproject.toml stubs uv.lockroot@eb913c2329bf:/workspace/scubaduck# grep -R "create_app(\"TEST\")" -n tests
tests/test_multi_table_web.py:14: app = create_app("TEST")I see that multi_table_web is used only there. I’ll check for any references in the tests.
root@eb913c2329bf:/workspace/scubaduck# sed -n '1,120p' tests/test_multi_table_w
eb.py
import threading
from collections.abc import Iterator
from typing import Any
import pytest
from werkzeug.serving import make_server
from scubaduck.server import create_app
from tests.test_web import select_value
@pytest.fixture()
def multi_table_server_url() -> Iterator[str]:
app = create_app("TEST")
httpd = make_server("127.0.0.1", 0, app)
port = httpd.server_port
thread = threading.Thread(target=httpd.serve_forever)
thread.start()
try:
yield f"http://127.0.0.1:{port}"
finally:
httpd.shutdown()
thread.join()
def test_table_param_updates_on_dive(page: Any, multi_table_server_url: str) ->
None:
page.goto(multi_table_server_url + "?table=events")
page.wait_for_selector("#table option", state="attached")
select_value(page, "#table", "extra")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
table_param = page.evaluate(
"new URLSearchParams(window.location.search).get('table')"
)
assert table_param == "extra"root@eb913c2329bf:/workspace/scubaduck# source .venv/bin/activate
(scubaduck) root@eb913c2329bf:/workspace/scubaduck#I need to run tests to verify the baseline functionality works as expected.
(scubaduck) root@eb913c2329bf:/workspace/scubaduck# pytest -qbringing up nodes........................................................................... [ 67%].................................. [100%]
106 passed in 14.46s(scubaduck) root@eb913c2329bf:/workspace/scubaduck# apply_patch<<'PATCH'
> *** Begin Patch
> *** Update File: scubaduck/server.py
> @@
> -from dataclasses import dataclass, field, replace
> -from typing import Any, Dict, List, Tuple, cast
> +from dataclasses import dataclass, field, replace
> +from typing import Any, Dict, List, Tuple, cast
> *** End Patch
> PATCH
Success. Updated the following files:
M scubaduck/server.py
(scubaduck) root@eb913c2329bf:/workspace/scubaduck#No changes were made, though the patch remains the same. I'll review it just to be sure.
(scubaduck) root@eb913c2329bf:/workspace/scubaduck# grep -n "_quote" -n scubaduc
k/server.py
(scubaduck) root@eb913c2329bf:/workspace/scubaduck#I’m moving forward with the implementation first, skipping quotes for now.
(scubaduck) root@eb913c2329bf:/workspace/scubaduck# sed -n '1,40p' scubaduck/ser
ver.py
from __future__ import annotations
from dataclasses import dataclass, field, replace
from typing import Any, Dict, List, Tuple, cast
import re
from datetime import datetime, timedelta, timezone
import time
from pathlib import Path
import os
import traceback
import math
import duckdb
from dateutil import parser as dtparser
from flask import Flask, jsonify, request, send_from_directory
@dataclass
class Filter:
column: str
op: str
value: str | int | float | list[str] | None
@dataclass
class QueryParams:
start: str | None = None
end: str | None = None
order_by: str | None = None
order_dir: str = "ASC"
limit: int | None = None
columns: list[str] = field(default_factory=lambda: [])
filters: list[Filter] = field(default_factory=lambda: [])
derived_columns: dict[str, str] = field(default_factory=lambda: {})
graph_type: str = "samples"
group_by: list[str] = field(default_factory=lambda: [])
aggregate: str | None = None
show_hits: bool = False
(scubaduck) root@eb913c2329bf:/workspace/scubaduck#I'll go ahead and add the _quote function right after the imports.
(scubaduck) root@eb913c2329bf:/workspace/scubaduck# apply_patch<<'PATCH'
> *** Begin Patch
> *** Update File: scubaduck/server.py
> @@
> -from flask import Flask, jsonify, request, send_from_directory
> +from flask import Flask, jsonify, request, send_from_directory
> +
> +
> +def _quote(ident: str) -> str:
> + """Return identifier quoted for SQL."""
> + return f'"{ident.replace("\"", "\"\"")}"'
> *** End Patch
> PATCH
Success. Updated the following files:
M scubaduck/server.py
(scubaduck) root@eb913c2329bf:/workspace/scubaduck#I'll make some changes to improve how _time_expr behaves.
(scubaduck) root@eb913c2329bf:/workspace/scubaduck# sed -n '148,220p' scubaduck/
server.py
"1 day": 86400,
"1 week": 604800,
"30 days": 2592000,
}
if gran in mapping:
return mapping[gran]
if gran in {"auto", "fine"} and start and end:
try:
s = dtparser.parse(start)
e = dtparser.parse(end)
except Exception:
return 3600
total = max((e - s).total_seconds(), 1)
buckets = 100 if gran == "auto" else 500
return max(int(total // buckets), 1)
return 3600
def _time_expr(col: str, column_types: Dict[str, str] | None, unit: str) -> str:
"""Return SQL expression for column interpreted as timestamp."""
if column_types is None:
return col
ctype = column_types.get(col, "").upper()
if not any(t in ctype for t in ["TIMESTAMP", "DATE", "TIME"]):
if any(
t in ctype
for t in [
"INT",
"DECIMAL",
"REAL",
"DOUBLE",
"FLOAT",
"NUMERIC",
"HUGEINT",
]
):
if unit == "ns":
expr = f"CAST({col} AS BIGINT)"
return f"make_timestamp_ns({expr})"
multiplier = {
"s": 1_000_000,
"ms": 1_000,
"us": 1,
}.get(unit, 1_000_000)
expr = (
f"CAST({col} * {multiplier} AS BIGINT)"
if multiplier != 1
else f"CAST({col} AS BIGINT)"
)
return f"make_timestamp({expr})"
return col
def build_query(params: QueryParams, column_types: Dict[str, str] | None = None)
-> str:
select_parts: list[str] = []
group_cols = params.group_by[:]
if params.graph_type == "timeseries":
sec = _granularity_seconds(params.granularity, params.start, params.end)
x_axis = params.x_axis or params.time_column
if x_axis is None:
raise ValueError("x_axis required for timeseries")
xexpr = _time_expr(x_axis, column_types, params.time_unit)
if params.start:
bucket_expr = (
f"TIMESTAMP '{params.start}' + INTERVAL '{sec} second' * "
f"CAST(floor((epoch({xexpr}) - epoch(TIMESTAMP '{params.start}')
)/{sec}) AS BIGINT)"
)
else:
bucket_expr = (
f"TIMESTAMP 'epoch' + INTERVAL '{sec} second' * "
f"CAST(floor(epoch({xexpr})/{sec}) AS BIGINT)"
)
(scubaduck) root@eb913c2329bf:/workspace/scubaduck#I'll adjust the code to fix it as needed.
(scubaduck) root@eb913c2329bf:/workspace/scubaduck# apply_patch<<'PATCH'
> *** Begin Patch
> *** Update File: scubaduck/server.py
> @@
> def _time_expr(col: str, column_types: Dict[str, str] | None, unit: str) -> s
tr:
> """Return SQL expression for column interpreted as timestamp."""
> - if column_types is None:
> - return col
> + qcol = _quote(col)
> + if column_types is None:
> + return qcol
> ctype = column_types.get(col, "").upper()
> if not any(t in ctype for t in ["TIMESTAMP", "DATE", "TIME"]):
> if any(
> t in ctype
> for t in [
> @@
> - expr = f"CAST({col} AS BIGINT)"
> + expr = f"CAST({qcol} AS BIGINT)"
> return f"make_timestamp_ns({expr})"
> @@
> - f"CAST({col} * {multiplier} AS BIGINT)"
> + f"CAST({qcol} * {multiplier} AS BIGINT)"
> if multiplier != 1
> - else f"CAST({col} AS BIGINT)"
> + else f"CAST({qcol} AS BIGINT)"
> )
> return f"make_timestamp({expr})"
> - return col
> + return qcol
> *** End Patch
> PATCH
Success. Updated the following files:
M scubaduck/server.py
(scubaduck) root@eb913c2329bf:/workspace/scubaduck#I'll modify build_query by quoting group columns and passing quoting into agg_expr.
(scubaduck) root@eb913c2329bf:/workspace/scubaduck# sed -n '220,360p' scubaduck/
server.py
f"CAST(floor(epoch({xexpr})/{sec}) AS BIGINT)"
)
select_parts.append(f"{bucket_expr} AS bucket")
group_cols = ["bucket"] + group_cols
has_agg = bool(group_cols) or params.aggregate is not None
if has_agg:
select_cols = (
group_cols[1:] if params.graph_type == "timeseries" else group_cols
)
select_parts.extend(select_cols)
agg = (params.aggregate or "count").lower()
def agg_expr(col: str) -> str:
expr = col
ctype = column_types.get(col, "").upper() if column_types else ""
if "BOOL" in ctype:
expr = f"CAST({col} AS BIGINT)"
if agg.startswith("p"):
quant = float(agg[1:]) / 100
return f"quantile({expr}, {quant})"
if agg == "count distinct":
return f"count(DISTINCT {expr})"
if agg == "avg" and column_types is not None:
if "TIMESTAMP" in ctype or "DATE" in ctype or "TIME" in ctype:
return (
"TIMESTAMP 'epoch' + INTERVAL '1 second' * "
f"CAST(avg(epoch({col})) AS BIGINT)"
)
return f"{agg}({expr})"
if agg == "count":
select_parts.append("count(*) AS Count")
else:
for col in params.columns:
if col in group_cols:
continue
select_parts.append(f"{agg_expr(col)} AS {col}")
if params.show_hits:
select_parts.insert(len(group_cols), "count(*) AS Hits")
else:
select_parts.extend(params.columns)
if has_agg and params.derived_columns:
inner_params = replace(
params,
derived_columns={},
order_by=None,
limit=None,
)
inner_sql = build_query(inner_params, column_types)
outer_select = ["t.*"] + [
f"{expr} AS {name}" for name, expr in params.derived_columns.items()
]
indented_inner = "\n".join(" " + line for line in inner_sql.splitline
s())
lines = [
f"SELECT {', '.join(outer_select)}",
"FROM (",
indented_inner,
") t",
]
if params.order_by:
lines.append(f"ORDER BY {params.order_by} {params.order_dir}")
elif params.graph_type == "timeseries":
lines.append("ORDER BY bucket")
if params.limit is not None:
lines.append(f"LIMIT {params.limit}")
return "\n".join(lines)
for name, expr in params.derived_columns.items():
select_parts.append(f"{expr} AS {name}")
select_clause = ", ".join(select_parts) if select_parts else "*"
lines = [f"SELECT {select_clause}", f'FROM "{params.table}"']
where_parts: list[str] = []
if params.time_column:
time_expr = _time_expr(params.time_column, column_types, params.time_uni
t)
else:
time_expr = None
if time_expr and params.start:
where_parts.append(f"{time_expr} >= '{params.start}'")
if time_expr and params.end:
where_parts.append(f"{time_expr} <= '{params.end}'")
for f in params.filters:
op = f.op
if op in {"empty", "!empty"}:
val = "''"
else:
if f.value is None:
continue
if isinstance(f.value, list):
if not f.value:
continue
if op == "=":
vals = " OR ".join(f"{f.column} = '{v}'" for v in f.value)
where_parts.append(f"({vals})")
continue
val = f"'{f.value}'" if isinstance(f.value, str) else str(f.value)
if op == "contains":
where_parts.append(f"{f.column} ILIKE '%' || {val} || '%'")
elif op == "!contains":
where_parts.append(f"{f.column} NOT ILIKE '%' || {val} || '%'")
elif op == "empty":
where_parts.append(f"{f.column} = {val}")
elif op == "!empty":
where_parts.append(f"{f.column} != {val}")
else:
where_parts.append(f"{f.column} {op} {val}")
if where_parts:
lines.append("WHERE " + " AND ".join(where_parts))
if group_cols:
lines.append("GROUP BY " + ", ".join(group_cols))
if params.order_by:
lines.append(f"ORDER BY {params.order_by} {params.order_dir}")
elif params.graph_type == "timeseries":
lines.append("ORDER BY bucket")
if params.limit is not None:
lines.append(f"LIMIT {params.limit}")
return "\n".join(lines)
def create_app(db_file: str | Path | None = None) -> Flask:
app = Flask(__name__, static_folder="static")
if db_file is None:
env_db = os.environ.get("SCUBADUCK_DB")
if env_db:
db_file = env_db
if isinstance(db_file, str) and db_file.upper() == "TEST":
con = _create_test_database()
else:
db_path = Path(db_file or Path(__file__).with_name("sample.csv")).resolv
e()
con = _load_database(db_path)
tables = [r[0] for r in con.execute("SHOW TABLES").fetchall()]
if not tables:
raise ValueError("No tables found in database")
default_table = tables[0]
columns_cache: Dict[str, Dict[str, str]] = {}
def get_columns(table: str) -> Dict[str, str]:
if table not in columns_cache:
rows = con.execute(f'PRAGMA table_info("{table}")').fetchall()
if not rows:
(scubaduck) root@eb913c2329bf:/workspace/scubaduck#Time to update the quoting logic and fix any issues that arise.
(scubaduck) root@eb913c2329bf:/workspace/scubaduck# apply_patch<<'PATCH'
> *** Begin Patch
> *** Update File: scubaduck/server.py
> @@
> - select_parts.extend(select_cols)
> + select_parts.extend(_quote(c) for c in select_cols)
> @@
> - expr = col
> + expr = _quote(col)
> ctype = column_types.get(col, "").upper() if column_types else ""
> if "BOOL" in ctype:
> - expr = f"CAST({col} AS BIGINT)"
> + expr = f"CAST({_quote(col)} AS BIGINT)"
> @@
> - f"CAST(avg(epoch({col})) AS BIGINT)"
> + f"CAST(avg(epoch({_quote(col)})) AS BIGINT)"
> @@
> - select_parts.append(f"{agg_expr(col)} AS {col}")
> + select_parts.append(f"{agg_expr(col)} AS {_quote(col)}")
> @@
> - select_parts.extend(params.columns)
> + select_parts.extend(_quote(c) for c in params.columns)
> *** End Patch
> PATCH
Success. Updated the following files:
M scubaduck/server.py
(scubaduck) root@eb913c2329bf:/workspace/scubaduck#I'm adding alias quoting and adjusting filtering, grouping, and ordering in the SQL query.
(scubaduck) root@eb913c2329bf:/workspace/scubaduck# sed -n '260,420p' scubaduck/
server.py
select_parts.extend(_quote(c) for c in params.columns)
if has_agg and params.derived_columns:
inner_params = replace(
params,
derived_columns={},
order_by=None,
limit=None,
)
inner_sql = build_query(inner_params, column_types)
outer_select = ["t.*"] + [
f"{expr} AS {name}" for name, expr in params.derived_columns.items()
]
indented_inner = "\n".join(" " + line for line in inner_sql.splitline
s())
lines = [
f"SELECT {', '.join(outer_select)}",
"FROM (",
indented_inner,
") t",
]
if params.order_by:
lines.append(f"ORDER BY {params.order_by} {params.order_dir}")
elif params.graph_type == "timeseries":
lines.append("ORDER BY bucket")
if params.limit is not None:
lines.append(f"LIMIT {params.limit}")
return "\n".join(lines)
for name, expr in params.derived_columns.items():
select_parts.append(f"{expr} AS {name}")
select_clause = ", ".join(select_parts) if select_parts else "*"
lines = [f"SELECT {select_clause}", f'FROM "{params.table}"']
where_parts: list[str] = []
if params.time_column:
time_expr = _time_expr(params.time_column, column_types, params.time_uni
t)
else:
time_expr = None
if time_expr and params.start:
where_parts.append(f"{time_expr} >= '{params.start}'")
if time_expr and params.end:
where_parts.append(f"{time_expr} <= '{params.end}'")
for f in params.filters:
op = f.op
if op in {"empty", "!empty"}:
val = "''"
else:
if f.value is None:
continue
if isinstance(f.value, list):
if not f.value:
continue
if op == "=":
vals = " OR ".join(f"{f.column} = '{v}'" for v in f.value)
where_parts.append(f"({vals})")
continue
val = f"'{f.value}'" if isinstance(f.value, str) else str(f.value)
if op == "contains":
where_parts.append(f"{f.column} ILIKE '%' || {val} || '%'")
elif op == "!contains":
where_parts.append(f"{f.column} NOT ILIKE '%' || {val} || '%'")
elif op == "empty":
where_parts.append(f"{f.column} = {val}")
elif op == "!empty":
where_parts.append(f"{f.column} != {val}")
else:
where_parts.append(f"{f.column} {op} {val}")
if where_parts:
lines.append("WHERE " + " AND ".join(where_parts))
if group_cols:
lines.append("GROUP BY " + ", ".join(group_cols))
if params.order_by:
lines.append(f"ORDER BY {params.order_by} {params.order_dir}")
elif params.graph_type == "timeseries":
lines.append("ORDER BY bucket")
if params.limit is not None:
lines.append(f"LIMIT {params.limit}")
return "\n".join(lines)
def create_app(db_file: str | Path | None = None) -> Flask:
app = Flask(__name__, static_folder="static")
if db_file is None:
env_db = os.environ.get("SCUBADUCK_DB")
if env_db:
db_file = env_db
if isinstance(db_file, str) and db_file.upper() == "TEST":
con = _create_test_database()
else:
db_path = Path(db_file or Path(__file__).with_name("sample.csv")).resolv
e()
con = _load_database(db_path)
tables = [r[0] for r in con.execute("SHOW TABLES").fetchall()]
if not tables:
raise ValueError("No tables found in database")
default_table = tables[0]
columns_cache: Dict[str, Dict[str, str]] = {}
def get_columns(table: str) -> Dict[str, str]:
if table not in columns_cache:
rows = con.execute(f'PRAGMA table_info("{table}")').fetchall()
if not rows:
raise ValueError(f"Unknown table: {table}")
columns_cache[table] = {r[1]: r[2] for r in rows}
return columns_cache[table]
sample_cache: Dict[Tuple[str, str, str], Tuple[List[str], float]] = {}
CACHE_TTL = 60.0
CACHE_LIMIT = 200
@app.route("/")
def index() -> Any: # pyright: ignore[reportUnusedFunction]
assert app.static_folder is not None
return send_from_directory(app.static_folder, "index.html")
@app.route("/js/<path:filename>")
def js(filename: str) -> Any: # pyright: ignore[reportUnusedFunction]
assert app.static_folder is not None
folder = Path(app.static_folder) / "js"
return send_from_directory(folder, filename)
@app.route("/api/tables")
def tables_endpoint() -> Any: # pyright: ignore[reportUnusedFunction]
return jsonify(tables)
@app.route("/api/columns")
def columns() -> Any: # pyright: ignore[reportUnusedFunction]
table = request.args.get("table", default_table)
rows = con.execute(f'PRAGMA table_info("{table}")').fetchall()
return jsonify([{"name": r[1], "type": r[2]} for r in rows])
def _cache_get(key: Tuple[str, str, str]) -> List[str] | None:
item = sample_cache.get(key)
if item is None:
return None
vals, ts = item
if time.time() - ts > CACHE_TTL:
del sample_cache[key]
return None
sample_cache[key] = (vals, time.time())
return vals
def _cache_set(key: Tuple[str, str, str], vals: List[str]) -> None:
sample_cache[key] = (vals, time.time())
if len(sample_cache) > CACHE_LIMIT:
oldest = min(sample_cache.items(), key=lambda kv: kv[1][1])[0]
del sample_cache[oldest]
@app.route("/api/samples")
def sample_values() -> Any: # pyright: ignore[reportUnusedFunction]
table = request.args.get("table", default_table)
column = request.args.get("column")
substr = request.args.get("q", "")
column_types = get_columns(table)
if not column or column not in column_types:
return jsonify([])
ctype = column_types[column].upper()
if "CHAR" not in ctype and "STRING" not in ctype and "VARCHAR" not in ct
ype:
return jsonify([])
key = (table, column, substr)
cached = _cache_get(key)
if cached is not None:
(scubaduck) root@eb913c2329bf:/workspace/scubaduck#Continuing to scroll past 420 lines to capture the rest.
(scubaduck) root@eb913c2329bf:/workspace/scubaduck# sed -n '420,640p' scubaduck/
server.py
if cached is not None:
return jsonify(cached)
rows = con.execute(
f"SELECT DISTINCT {column} FROM \"{table}\" WHERE CAST({column} AS V
ARCHAR) ILIKE '%' || ? || '%' LIMIT 20",
[substr],
).fetchall()
values = [r[0] for r in rows]
_cache_set(key, values)
return jsonify(values)
@app.route("/api/query", methods=["POST"])
def query() -> Any: # pyright: ignore[reportUnusedFunction]
payload = request.get_json(force=True)
try:
start = parse_time(payload.get("start"))
end = parse_time(payload.get("end"))
except Exception as exc:
return jsonify({"error": str(exc)}), 400
params = QueryParams(
start=start,
end=end,
order_by=payload.get("order_by"),
order_dir=payload.get("order_dir", "ASC"),
limit=payload.get("limit"),
columns=payload.get("columns", []),
derived_columns=payload.get("derived_columns", {}),
graph_type=payload.get("graph_type", "samples"),
group_by=payload.get("group_by", []),
aggregate=payload.get("aggregate"),
show_hits=payload.get("show_hits", False),
x_axis=payload.get("x_axis"),
granularity=payload.get("granularity", "Auto"),
fill=payload.get("fill", "0"),
table=payload.get("table", default_table),
time_column=payload.get("time_column", "timestamp"),
time_unit=payload.get("time_unit", "s"),
)
for f in payload.get("filters", []):
params.filters.append(Filter(f["column"], f["op"], f.get("value")))
if params.table not in tables:
return jsonify({"error": "Invalid table"}), 400
column_types = get_columns(params.table)
if params.time_column and params.time_column not in column_types:
return jsonify({"error": "Invalid time_column"}), 400
if params.time_unit not in {"s", "ms", "us", "ns"}:
return jsonify({"error": "Invalid time_unit"}), 400
if params.graph_type not in {"table", "timeseries"} and (
params.group_by or params.aggregate or params.show_hits
):
return (
jsonify(
{
"error": "group_by, aggregate and show_hits are only val
id for table or timeseries view"
}
),
400,
)
valid_cols = set(column_types.keys())
valid_cols.update(params.derived_columns.keys())
if params.graph_type == "timeseries":
if params.x_axis is None:
params.x_axis = params.time_column
if params.x_axis is None or params.x_axis not in valid_cols:
return jsonify({"error": "Invalid x_axis"}), 400
ctype = column_types.get(params.x_axis, "").upper()
is_time = any(t in ctype for t in ["TIMESTAMP", "DATE", "TIME"])
is_numeric = any(
t in ctype
for t in [
"INT",
"DECIMAL",
"REAL",
"DOUBLE",
"FLOAT",
"NUMERIC",
"HUGEINT",
]
)
if not (is_time or is_numeric):
return jsonify({"error": "x_axis must be a time column"}), 400
for col in params.columns:
if col not in valid_cols:
return jsonify({"error": f"Unknown column: {col}"}), 400
for col in params.group_by:
if col not in valid_cols:
return jsonify({"error": f"Unknown column: {col}"}), 400
if params.order_by and params.order_by not in valid_cols:
return jsonify({"error": f"Unknown column: {params.order_by}"}), 400
if params.group_by or params.graph_type == "timeseries":
agg = (params.aggregate or "count").lower()
if agg.startswith("p") or agg == "sum":
need_numeric = True
allow_time = False
elif agg == "avg" or agg in {"min", "max"}:
need_numeric = False
allow_time = True
else:
need_numeric = False
allow_time = False
if need_numeric or allow_time:
for c in params.columns:
if c in params.group_by or c == params.x_axis:
continue
if c not in column_types:
continue
ctype = column_types.get(c, "").upper()
is_numeric = any(
t in ctype
for t in [
"INT",
"DECIMAL",
"REAL",
"DOUBLE",
"FLOAT",
"NUMERIC",
"HUGEINT",
]
)
is_time = "TIMESTAMP" in ctype or "DATE" in ctype or "TIME"
in ctype
if need_numeric and not is_numeric:
return (
jsonify(
{
"error": f"Aggregate {agg} cannot be applied
to column {c}",
}
),
400,
)
if allow_time and not (is_numeric or is_time):
return (
jsonify(
{
"error": f"Aggregate {agg} cannot be applied
to column {c}",
}
),
400,
)
if (params.start is None or params.end is None) and (
params.x_axis or params.time_column
):
axis = params.x_axis or params.time_column
row = cast(
tuple[datetime | None, datetime | None],
con.execute(
f'SELECT min({axis}), max({axis}) FROM "{params.table}"'
).fetchall()[0],
)
mn, mx = row
divisor = {
"s": 1,
"ms": 1000,
"us": 1_000_000,
"ns": 1_000_000_000,
}.get(params.time_unit, 1)
if isinstance(mn, (int, float)):
try:
mn = datetime.fromtimestamp(int(mn) / divisor, tz=timezone.u
tc)
except Exception:
return (
jsonify(
{
"error": (
f"Invalid time value {mn} for column {axis}"
f" with time_unit {params.time_unit}"
)
}
),
400,
)
if isinstance(mx, (int, float)):
try:
mx = datetime.fromtimestamp(int(mx) / divisor, tz=timezone.u
tc)
except Exception:
return (
jsonify(
{
"error": (
f"Invalid time value {mx} for column {axis}"
f" with time_unit {params.time_unit}"
)
}
),
400,
)
if params.start is None and mn is not None:
params.start = (
mn.strftime("%Y-%m-%d %H:%M:%S") if not isinstance(mn, str)
else mn
)
if params.end is None and mx is not None:
params.end = (
mx.strftime("%Y-%m-%d %H:%M:%S") if not isinstance(mx, str)
else mx
)
bucket_size: int | None = None
if params.graph_type == "timeseries":
bucket_size = _granularity_seconds(
params.granularity,
params.start if isinstance(params.start, str) else None,
params.end if isinstance(params.end, str) else None,
)
if (
params.limit is not None
and params.start is not None
and params.end is not None
):
try:
start_dt = dtparser.parse(params.start)
end_dt = dtparser.parse(params.end)
buckets = math.ceil(
(end_dt - start_dt).total_seconds() / bucket_size
)
if buckets > 1:
params.limit *= buckets
(scubaduck) root@eb913c2329bf:/workspace/scubaduck#I need to patch _cache_s and sample_values to add quoting.
(scubaduck) root@eb913c2329bf:/workspace/scubaduck# apply_patch<<'PATCH'
> *** Begin Patch
> *** Update File: scubaduck/server.py
> @@
> - rows = con.execute(
> - f"SELECT DISTINCT {column} FROM \"{table}\" WHERE CAST({column} A
S VARCHAR) ILIKE '%' || ? || '%' LIMIT 20",
> - [substr],
> - ).fetchall()
> + qcol = _quote(column)
> + rows = con.execute(
> + f"SELECT DISTINCT {qcol} FROM \"{table}\" WHERE CAST({qcol} AS VA
RCHAR) ILIKE '%' || ? || '%' LIMIT 20",
> + [substr],
> + ).fetchall()
> *** End Patch
> PATCH
Success. Updated the following files:
M scubaduck/server.py
(scubaduck) root@eb913c2329bf:/workspace/scubaduck#Time to patch order_by and any necessary related updates. Let's get started!
(scubaduck) root@eb913c2329bf:/workspace/scubaduck# apply_patch<<'PATCH'
> *** Begin Patch
> *** Update File: scubaduck/server.py
> @@
> - if params.order_by:
> - lines.append(f"ORDER BY {params.order_by} {params.order_dir}")
> + if params.order_by:
> + lines.append(f"ORDER BY {_quote(params.order_by)} {params.order_d
ir}")
> *** End Patch
> PATCH
Success. Updated the following files:
M scubaduck/server.py
(scubaduck) root@eb913c2329bf:/workspace/scubaduck#I'll check the query for any missing patch related to "ORDER BY {params.order_by}".
(scubaduck) root@eb913c2329bf:/workspace/scubaduck# grep -n "ORDER BY" -n scubad
uck/server.py
281: lines.append(f"ORDER BY {_quote(params.order_by)} {params.order_
dir}")
283: lines.append("ORDER BY bucket")
332: lines.append(f"ORDER BY {params.order_by} {params.order_dir}")
334: lines.append("ORDER BY bucket")
(scubaduck) root@eb913c2329bf:/workspace/scubaduck#I’ll patch at line 332 as well, alongside the change at line 281.
(scubaduck) root@eb913c2329bf:/workspace/scubaduck# sed -n '320,360p' scubaduck/
server.py
where_parts.append(f"{f.column} NOT ILIKE '%' || {val} || '%'")
elif op == "empty":
where_parts.append(f"{f.column} = {val}")
elif op == "!empty":
where_parts.append(f"{f.column} != {val}")
else:
where_parts.append(f"{f.column} {op} {val}")
if where_parts:
lines.append("WHERE " + " AND ".join(where_parts))
if group_cols:
lines.append("GROUP BY " + ", ".join(group_cols))
if params.order_by:
lines.append(f"ORDER BY {params.order_by} {params.order_dir}")
elif params.graph_type == "timeseries":
lines.append("ORDER BY bucket")
if params.limit is not None:
lines.append(f"LIMIT {params.limit}")
return "\n".join(lines)
def create_app(db_file: str | Path | None = None) -> Flask:
app = Flask(__name__, static_folder="static")
if db_file is None:
env_db = os.environ.get("SCUBADUCK_DB")
if env_db:
db_file = env_db
if isinstance(db_file, str) and db_file.upper() == "TEST":
con = _create_test_database()
else:
db_path = Path(db_file or Path(__file__).with_name("sample.csv")).resolv
e()
con = _load_database(db_path)
tables = [r[0] for r in con.execute("SHOW TABLES").fetchall()]
if not tables:
raise ValueError("No tables found in database")
default_table = tables[0]
columns_cache: Dict[str, Dict[str, str]] = {}
def get_columns(table: str) -> Dict[str, str]:
if table not in columns_cache:
rows = con.execute(f'PRAGMA table_info("{table}")').fetchall()
if not rows:(scubaduck) root@eb913c2329bf:/workspace/scubaduck# apply_patch<<'PATCH'
> *** Begin Patch
> *** Update File: scubaduck/server.py
> @@
> - if params.order_by:
> - lines.append(f"ORDER BY {params.order_by} {params.order_dir}")
> + if params.order_by:
> + lines.append(f"ORDER BY {_quote(params.order_by)} {params.order_dir}"
)
> *** End Patch
> PATCH
Success. Updated the following files:
M scubaduck/server.py
(scubaduck) root@eb913c2329bf:/workspace/scubaduck#Time to patch group_cols and filters. I'll add the GROUP BY line and quote.
(scubaduck) root@eb913c2329bf:/workspace/scubaduck# apply_patch<<'PATCH'
> *** Begin Patch
> *** Update File: scubaduck/server.py
> @@
> - if group_cols:
> - lines.append("GROUP BY " + ", ".join(group_cols))
> + if group_cols:
> + lines.append("GROUP BY " + ", ".join(_quote(c) for c in group_cols))
> *** End Patch
> PATCH
Success. Updated the following files:
M scubaduck/server.py
(scubaduck) root@eb913c2329bf:/workspace/scubaduck#I'll work on updating the filter conditions and search for the right changes.
(scubaduck) root@eb913c2329bf:/workspace/scubaduck# grep -n "ILIKE" -n scubaduck
/server.py
318: where_parts.append(f"{f.column} ILIKE '%' || {val} || '%'")
320: where_parts.append(f"{f.column} NOT ILIKE '%' || {val} || '%'")
424: f"SELECT DISTINCT {qcol} FROM \"{table}\" WHERE CAST({qcol} AS V
ARCHAR) ILIKE '%' || ? || '%' LIMIT 20",
(scubaduck) root@eb913c2329bf:/workspace/scubaduck#I'll focus on lines 318, 320, etc., and modify the filter loops.
(scubaduck) root@eb913c2329bf:/workspace/scubaduck# sed -n '306,338p' scubaduck/
server.py
if f.value is None:
continue
if isinstance(f.value, list):
if not f.value:
continue
if op == "=":
vals = " OR ".join(f"{f.column} = '{v}'" for v in f.value)
where_parts.append(f"({vals})")
continue
val = f"'{f.value}'" if isinstance(f.value, str) else str(f.value)
if op == "contains":
where_parts.append(f"{f.column} ILIKE '%' || {val} || '%'")
elif op == "!contains":
where_parts.append(f"{f.column} NOT ILIKE '%' || {val} || '%'")
elif op == "empty":
where_parts.append(f"{f.column} = {val}")
elif op == "!empty":
where_parts.append(f"{f.column} != {val}")
else:
where_parts.append(f"{f.column} {op} {val}")
if where_parts:
lines.append("WHERE " + " AND ".join(where_parts))
if group_cols:
lines.append("GROUP BY " + ", ".join(_quote(c) for c in group_cols))
if params.order_by:
lines.append(f"ORDER BY {_quote(params.order_by)} {params.order_dir}")
elif params.graph_type == "timeseries":
lines.append("ORDER BY bucket")
if params.limit is not None:
lines.append(f"LIMIT {params.limit}")
return "\n".join(lines)
(scubaduck) root@eb913c2329bf:/workspace/scubaduck#I’m going to fix this quoting issue now.
(scubaduck) root@eb913c2329bf:/workspace/scubaduck# apply_patch<<'PATCH'
> *** Begin Patch
> *** Update File: scubaduck/server.py
> @@
> - if op == "=":
> - vals = " OR ".join(f"{f.column} = '{v}'" for v in f.value
)
> - where_parts.append(f"({vals})")
> + if op == "=":
> + qcol = _quote(f.column)
> + vals = " OR ".join(f"{qcol} = '{v}'" for v in f.value)
> + where_parts.append(f"({vals})")
> continue
> @@
> - if op == "contains":
> - where_parts.append(f"{f.column} ILIKE '%' || {val} || '%'")
> - elif op == "!contains":
> - where_parts.append(f"{f.column} NOT ILIKE '%' || {val} || '%'")
> - elif op == "empty":
> - where_parts.append(f"{f.column} = {val}")
> - elif op == "!empty":
> - where_parts.append(f"{f.column} != {val}")
> - else:
> - where_parts.append(f"{f.column} {op} {val}")
> + qcol = _quote(f.column)
> + if op == "contains":
> + where_parts.append(f"{qcol} ILIKE '%' || {val} || '%'")
> + elif op == "!contains":
> + where_parts.append(f"{qcol} NOT ILIKE '%' || {val} || '%'")
> + elif op == "empty":
> + where_parts.append(f"{qcol} = {val}")
> + elif op == "!empty":
> + where_parts.append(f"{qcol} != {val}")
> + else:
> + where_parts.append(f"{qcol} {op} {val}")
> *** End Patch
> PATCH
Success. Updated the following files:
M scubaduck/server.py
(scubaduck) root@eb913c2329bf:/workspace/scubaduck#I changed filter loops and need to patch section where column names need quoting.
(scubaduck) root@eb913c2329bf:/workspace/scubaduck# grep -n "SELECT min" -n scub
aduck/server.py
575: f'SELECT min({axis}), max({axis}) FROM "{params.table}"'
(scubaduck) root@eb913c2329bf:/workspace/scubaduck#I need to address the issue with the quoting. I'll go ahead and patch it.
(scubaduck) root@eb913c2329bf:/workspace/scubaduck# sed -n '560,584p' scubaduck/
server.py
return (
jsonify(
{
"error": f"Aggregate {agg} cannot be applied
to column {c}",
}
),
400,
)
if (params.start is None or params.end is None) and (
params.x_axis or params.time_column
):
axis = params.x_axis or params.time_column
row = cast(
tuple[datetime | None, datetime | None],
con.execute(
f'SELECT min({axis}), max({axis}) FROM "{params.table}"'
).fetchall()[0],
)
mn, mx = row
divisor = {
"s": 1,
"ms": 1000,
"us": 1_000_000,
"ns": 1_000_000_000,
}.get(params.time_unit, 1)
(scubaduck) root@eb913c2329bf:/workspace/scubaduck#Time to apply the patch!
(scubaduck) root@eb913c2329bf:/workspace/scubaduck# apply_patch<<'PATCH'
> *** Begin Patch
> *** Update File: scubaduck/server.py
> @@
> - axis = params.x_axis or params.time_column
> + axis = params.x_axis or params.time_column
> row = cast(
> tuple[datetime | None, datetime | None],
> con.execute(
> - f'SELECT min({axis}), max({axis}) FROM "{params.table}"'
> + f'SELECT min({_quote(axis)}), max({_quote(axis)}) FROM "{
params.table}"'
> ).fetchall()[0],
> )
> *** End Patch
> PATCH
Success. Updated the following files:
M scubaduck/server.py
(scubaduck) root@eb913c2329bf:/workspace/scubaduck#It looks like the quoting fixes are mostly in place; only minor adjustments to group_cols remain.
(scubaduck) root@eb913c2329bf:/workspace/scubaduck# ruff format scubaduck/server
.py
1 file reformatted(scubaduck) root@eb913c2329bf:/workspace/scubaduck# ruff check scubaduck/server.
py
All checks passed!(scubaduck) root@eb913c2329bf:/workspace/scubaduck# pyright scubaduck/server.py
> /tmp/pyright.log && tail -n 20 /tmp/pyright.log(scubaduck) root@eb913c2329bf:/workspace/scubaduck#I suspect the missing newline suppresses pyright output, so I'll cat the log.
(scubaduck) root@eb913c2329bf:/workspace/scubaduck# cat /tmp/pyright.log
/workspace/scubaduck/scubaduck/server.py
/workspace/scubaduck/scubaduck/server.py:575:42 - error: Argument of type "str
| None" cannot be assigned to parameter "ident" of type "str" in function "_quo
te"
Type "str | None" is not assignable to type "str"
"None" is not assignable to "str" (reportArgumentType)
/workspace/scubaduck/scubaduck/server.py:575:63 - error: Argument of type "str
| None" cannot be assigned to parameter "ident" of type "str" in function "_quo
te"
Type "str | None" is not assignable to type "str"
"None" is not assignable to "str" (reportArgumentType)
2 errors, 0 warnings, 0 informations
(scubaduck) root@eb913c2329bf:/workspace/scubaduck#I’ll assert axis is not None before passing it to _quote to resolve the pyright issue.
(scubaduck) root@eb913c2329bf:/workspace/scubaduck# sed -n '568,584p' scubaduck/
server.py
if (params.start is None or params.end is None) and (
params.x_axis or params.time_column
):
axis = params.x_axis or params.time_column
row = cast(
tuple[datetime | None, datetime | None],
con.execute(
f'SELECT min({_quote(axis)}), max({_quote(axis)}) FROM "{par
ams.table}"'
).fetchall()[0],
)
mn, mx = row
divisor = {
"s": 1,
"ms": 1000,
"us": 1_000_000,
"ns": 1_000_000_000,
}.get(params.time_unit, 1)(scubaduck) root@eb913c2329bf:/workspace/scubaduck# apply_patch<<'PATCH'
> *** Begin Patch
> *** Update File: scubaduck/server.py
> @@
> if (params.start is None or params.end is None) and (
> params.x_axis or params.time_column
> ):
> - axis = params.x_axis or params.time_column
> + axis = params.x_axis or params.time_column
> + assert axis is not None
> row = cast(
> tuple[datetime | None, datetime | None],
> con.execute(
> f'SELECT min({_quote(axis)}), max({_quote(axis)}) FROM "{
params.table}"'
> ).fetchall()[0],
> *** End Patch
> PATCH
Success. Updated the following files:
M scubaduck/server.py(scubaduck) root@eb913c2329bf:/workspace/scubaduck# ruff format scubaduck/server
.py
1 file left unchanged
(scubaduck) root@eb913c2329bf:/workspace/scubaduck#I'll try running the ruff check again to see if issues persist.
(scubaduck) root@eb913c2329bf:/workspace/scubaduck# ruff check scubaduck/server.
py
All checks passed!(scubaduck) root@eb913c2329bf:/workspace/scubaduck# pyright scubaduck/server.py
> /tmp/pyright.log && tail -n 20 /tmp/pyright.log0 errors, 0 warnings, 0 informations
(scubaduck) root@eb913c2329bf:/workspace/scubaduck#I added _quote to sample_values. Maybe it's a good idea to run a ruff check.
(scubaduck) root@eb913c2329bf:/workspace/scubaduck# ruff check
All checks passed!
(scubaduck) root@eb913c2329bf:/workspace/scubaduck#I'm running Pyright for type checking across the entire repository.
(scubaduck) root@eb913c2329bf:/workspace/scubaduck# pyright > /tmp/pyright.log &
& tail -n 20 /tmp/pyright.log0 errors, 0 warnings, 0 informations
(scubaduck) root@eb913c2329bf:/workspace/scubaduck#I'll add a new test to verify the quoting behavior in test_server.py. Checking placement.
(scubaduck) root@eb913c2329bf:/workspace/scubaduck# grep -n "_make_payload" -n t
ests/test_server.py
157:def _make_payload() -> dict[str, object]:
201: payload = _make_payload()
427: payload = _make_payload()
(scubaduck) root@eb913c2329bf:/workspace/scubaduck#I'll check the file between lines 420 and 460 to inspect the bottom section.
.py
)
data = rv.get_json()
assert rv.status_code == 200
rows = data["rows"]
assert all(r[2] == r[1] * 2 for r in rows)
def test_default_start_end_returned() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"order_by": "timestamp",
"limit": 5,
"columns": ["timestamp"],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert data["start"] == "2024-01-01 00:00:00"
assert data["end"] == "2024-01-02 03:00:00"
def test_time_column_none_no_time_filter() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"columns": ["timestamp", "event"],
"time_column": "",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert "start" not in data and "end" not in data
assert len(data["rows"]) == 4
(scubaduck) root@eb913c2329bf:/workspace/scubaduck#I'll add a check for the data inserted in _create_test_database() and validate rows.
> *** Begin Patch
> *** Update File: tests/test_server.py
> @@
> def test_time_column_none_no_time_filter() -> None:
> @@
> assert len(data["rows"]) == 4
> +
> +
> +def test_reserved_word_column() -> None:
> + app = server.create_app("TEST")
> + client = app.test_client()
> + payload = {
> + "table": "extra",
> + "columns": ["ts", "desc"],
> + "order_by": "ts",
> + }
> + rv = client.post(
> + "/api/query", data=json.dumps(payload), content_type="application/jso
n"
> + )
> + data = rv.get_json()
> + assert rv.status_code == 200
> + assert len(data["rows"]) == 2
> + assert data["rows"][0][1] == "x"
> *** End Patch
> PATCH
Success. Updated the following files:
M tests/test_server.pyr.py
1 file left unchanged
(scubaduck) root@eb913c2329bf:/workspace/scubaduck#I'll check the tests using ruff!
(scubaduck) root@eb913c2329bf:/workspace/scubaduck# ruff check tests/test_server
.py
All checks passed!
(scubaduck) root@eb913c2329bf:/workspace/scubaduck#I'll check the entire repository with ruff now.
(scubaduck) root@eb913c2329bf:/workspace/scubaduck# ruff check
All checks passed!
(scubaduck) root@eb913c2329bf:/workspace/scubaduck#I'll go ahead and run pyright again to double-check everything.
(scubaduck) root@eb913c2329bf:/workspace/scubaduck# pyright > /tmp/pyright.log &
& tail -n 20 /tmp/pyright.log0 errors, 0 warnings, 0 informations(scubaduck) root@eb913c2329bf:/workspace/scubaduck# pytest -qbringing up nodes........................................................................... [ 67%]..............................F.... [100%]
=================================== FAILURES ===================================
__________________________ test_reserved_word_column ___________________________
[gw2] linux -- Python 3.12.3 /workspace/scubaduck/.venv/bin/python
Traceback (most recent call last):
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/runner.p
y", line 341, in from_call
result: TResult | None = func()
^^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/runner.p
y", line 242, in <lambda>
lambda: runtest_hook(item=item, **kwds), when=when, reraise=reraise
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_hooks.py
", line 512, in __call__
return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult
)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_manager.
py", line 120, in _hookexec
return self._inner_hookexec(hook_name, methods, kwargs, firstresult)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_callers.
py", line 167, in _multicall
raise exception
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_callers.
py", line 139, in _multicall
teardown.throw(exception)
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/threadex
ception.py", line 92, in pytest_runtest_call
yield from thread_exception_runtest_hook()
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/threadex
ception.py", line 68, in thread_exception_runtest_hook
yield
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_callers.
py", line 139, in _multicall
teardown.throw(exception)
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/unraisab
leexception.py", line 95, in pytest_runtest_call
yield from unraisable_exception_runtest_hook()
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/unraisab
leexception.py", line 70, in unraisable_exception_runtest_hook
yield
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_callers.
py", line 139, in _multicall
teardown.throw(exception)
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/logging.
py", line 846, in pytest_runtest_call
yield from self._runtest_for(item, "call")
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/logging.
py", line 829, in _runtest_for
yield
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_callers.
py", line 139, in _multicall
teardown.throw(exception)
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/capture.
py", line 898, in pytest_runtest_call
return (yield)
^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_callers.
py", line 139, in _multicall
teardown.throw(exception)
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/skipping
.py", line 257, in pytest_runtest_call
return (yield)
^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_callers.
py", line 121, in _multicall
res = hook_impl.function(*args)
^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/runner.p
y", line 174, in pytest_runtest_call
item.runtest()
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/python.p
y", line 1627, in runtest
self.ihook.pytest_pyfunc_call(pyfuncitem=self)
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_hooks.py
", line 512, in __call__
return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult
)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_manager.
py", line 120, in _hookexec
return self._inner_hookexec(hook_name, methods, kwargs, firstresult)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_callers.
py", line 167, in _multicall
raise exception
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_callers.
py", line 121, in _multicall
res = hook_impl.function(*args)
^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/python.p
y", line 159, in pytest_pyfunc_call
result = testfunction(**testargs)
^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/scubaduck/tests/test_server.py", line 844, in test_reserved_w
ord_column
assert rv.status_code == 200
AssertionError: assert 400 == 200
+ where 400 = <WrapperTestResponse 32 bytes [400 BAD REQUEST]>.status_code
=========================== short test summary info ============================
FAILED tests/test_server.py::test_reserved_word_column - assert 400 == 200
1 failed, 106 passed in 13.89s
(scubaduck) root@eb913c2329bf:/workspace/scubaduck#It seems there was a misunderstanding with the "desc" column causing a 400 error.
(scubaduck) root@eb913c2329bf:/workspace/scubaduck# python - <<'PY'
> import json
> from scubaduck import server
> app = server.create_app("TEST")
> client = app.test_client()
> payload={"table":"extra","columns":["ts","desc"],"order_by":"ts"}
> rv=client.post("/api/query", data=json.dumps(payload), content_type="applicati
on/json")
> print(rv.status_code)
> print(rv.get_json())
> PY400
{'error': 'Invalid time_column'}
(scubaduck) root@eb913c2329bf:/workspace/scubaduck#I'm addressing the missing "timestamp" column in "extra". Time column should be set to "".
(scubaduck) root@eb913c2329bf:/workspace/scubaduck# apply_patch<<'PATCH'
> *** Begin Patch
> *** Update File: tests/test_server.py
> @@
> - payload = {
> - "table": "extra",
> - "columns": ["ts", "desc"],
> - "order_by": "ts",
> - }
> + payload = {
> + "table": "extra",
> + "columns": ["ts", "desc"],
> + "order_by": "ts",
> + "time_column": "",
> + }
> *** End Patch
> PATCH
Success. Updated the following files:
M tests/test_server.py(scubaduck) root@eb913c2329bf:/workspace/scubaduck# ruff format tests/test_serve
r.py
1 file left unchanged(scubaduck) root@eb913c2329bf:/workspace/scubaduck# ruff check tests/test_server
.py
All checks passed!
(scubaduck) root@eb913c2329bf:/workspace/scubaduck#Time to run the complete ruff check to ensure everything is in order.
(scubaduck) root@eb913c2329bf:/workspace/scubaduck# ruff check
All checks passed!(scubaduck) root@eb913c2329bf:/workspace/scubaduck# pyright > /tmp/pyright.log &
& tail -n 20 /tmp/pyright.log0 errors, 0 warnings, 0 informations
(scubaduck) root@eb913c2329bf:/workspace/scubaduck#Time to give the tests another shot!
(scubaduck) root@eb913c2329bf:/workspace/scubaduck# pytest -qbringing up nodes........................................................................... [ 67%]................................... [100%]
107 passed in 14.55s
(scubaduck) root@eb913c2329bf:/workspace/scubaduck#All tests passed! I'll check git status before committing the changes.
(scubaduck) root@eb913c2329bf:/workspace/scubaduck# git status --short
M scubaduck/server.py
M tests/test_server.py
(scubaduck) root@eb913c2329bf:/workspace/scubaduck#I'll take a closer look at the differences.
(scubaduck) root@eb913c2329bf:/workspace/scubaduck# git diff --stat
scubaduck/server.py | 54 +++++++++++++++++++++++++++++++---------------------
tests/test_server.py | 18 ++++++++++++++++++
2 files changed, 50 insertions(+), 22 deletions(-)
(scubaduck) root@eb913c2329bf:/workspace/scubaduck#I'm inspecting the diff to ensure the proper quoting format.
diff --git a/scubaduck/server.py b/scubaduck/server.py
index c9b7bd1..7c7a215 100644
--- a/scubaduck/server.py
+++ b/scubaduck/server.py
@@ -17,6 +17,11 @@ from dateutil import parser as dtparser
from flask import Flask, jsonify, request, send_from_directory
+def _quote(ident: str) -> str:
+ """Return identifier quoted for SQL."""
+ return f'"{ident.replace('"', '""')}"'
+
+
@dataclass
class Filter:
column: str
@@ -160,8 +165,9 @@ def _granularity_seconds(granularity: str, start: str | None
, end: str | None) -
def _time_expr(col: str, column_types: Dict[str, str] | None, unit: str) -> str
:
"""Return SQL expression for column interpreted as timestamp."""
+ qcol = _quote(col)
if column_types is None:
- return col
+ return qcol
ctype = column_types.get(col, "").upper()
if not any(t in ctype for t in ["TIMESTAMP", "DATE", "TIME"]):
if any(
@@ -177,7 +183,7 @@ def _time_expr(col: str, column_types: Dict[str, str] | None
, unit: str) -> str:
]
):
if unit == "ns":
- expr = f"CAST({col} AS BIGINT)"
+ expr = f"CAST({qcol} AS BIGINT)"
return f"make_timestamp_ns({expr})"
multiplier = {
@@ -186,12 +192,12 @@ def _time_expr(col: str, column_types: Dict[str, str] | No
ne, unit: str) -> str:
"us": 1,
}.get(unit, 1_000_000)
expr = (| sed -n '40,160p'
expr = (
- f"CAST({col} * {multiplier} AS BIGINT)"
+ f"CAST({qcol} * {multiplier} AS BIGINT)"
if multiplier != 1
- else f"CAST({col} AS BIGINT)"
+ else f"CAST({qcol} AS BIGINT)"
)
return f"make_timestamp({expr})"
- return col
+ return qcol
def build_query(params: QueryParams, column_types: Dict[str, str] | None = None
) -> str:
@@ -220,14 +226,14 @@ def build_query(params: QueryParams, column_types: Dict[st
r, str] | None = None)
select_cols = (
group_cols[1:] if params.graph_type == "timeseries" else group_cols
)
- select_parts.extend(select_cols)
+ select_parts.extend(_quote(c) for c in select_cols)
agg = (params.aggregate or "count").lower()
def agg_expr(col: str) -> str:
- expr = col
+ expr = _quote(col)
ctype = column_types.get(col, "").upper() if column_types else ""
if "BOOL" in ctype:
- expr = f"CAST({col} AS BIGINT)"
+ expr = f"CAST({_quote(col)} AS BIGINT)"
if agg.startswith("p"):
quant = float(agg[1:]) / 100
return f"quantile({expr}, {quant})"
@@ -237,7 +243,7 @@ def build_query(params: QueryParams, column_types: Dict[str,
str] | None = None)
if "TIMESTAMP" in ctype or "DATE" in ctype or "TIME" in ctype:
return (
"TIMESTAMP 'epoch' + INTERVAL '1 second' * "
- f"CAST(avg(epoch({col})) AS BIGINT)"
+ f"CAST(avg(epoch({_quote(col)})) AS BIGINT)"
)
return f"{agg}({expr})"
@@ -247,11 +253,11 @@ def build_query(params: QueryParams, column_types: Dict[st
r, str] | None = None)
for col in params.columns:
if col in group_cols:
continue
- select_parts.append(f"{agg_expr(col)} AS {col}")
+ select_parts.append(f"{agg_expr(col)} AS {_quote(col)}")
if params.show_hits:
select_parts.insert(len(group_cols), "count(*) AS Hits")
else:
- select_parts.extend(params.columns)
+ select_parts.extend(_quote(c) for c in params.columns)
if has_agg and params.derived_columns:
inner_params = replace(
@@ -272,7 +278,7 @@ def build_query(params: QueryParams, column_types: Dict[str,
str] | None = None)
") t",
]
if params.order_by:
- lines.append(f"ORDER BY {params.order_by} {params.order_dir}")
+ lines.append(f"ORDER BY {_quote(params.order_by)} {params.order_dir
}")
elif params.graph_type == "timeseries":
lines.append("ORDER BY bucket")
if params.limit is not None:
@@ -303,27 +309,29 @@ def build_query(params: QueryParams, column_types: Dict[st
r, str] | None = None)
if not f.value:
continue
if op == "=":
- vals = " OR ".join(f"{f.column} = '{v}'" for v in f.value)
+ qcol = _quote(f.column)
+ vals = " OR ".join(f"{qcol} = '{v}'" for v in f.value)
where_parts.append(f"({vals})")
continue
val = f"'{f.value}'" if isinstance(f.value, str) else str(f.value)
+ qcol = _quote(f.column)
if op == "contains":
- where_parts.append(f"{f.column} ILIKE '%' || {val} || '%'")
+ where_parts.append(f"{qcol} ILIKE '%' || {val} || '%'")
elif op == "!contains":
- where_parts.append(f"{f.column} NOT ILIKE '%' || {val} || '%'")
+ where_parts.append(f"{qcol} NOT ILIKE '%' || {val} || '%'")
elif op == "empty":
- where_parts.append(f"{f.column} = {val}")
+ where_parts.append(f"{qcol} = {val}")
elif op == "!empty":
- where_parts.append(f"{f.column} != {val}")
+ where_parts.append(f"{qcol} != {val}")
else:
- where_parts.append(f"{f.column} {op} {val}")
+ where_parts.append(f"{qcol} {op} {val}")
if where_parts:
lines.append("WHERE " + " AND ".join(where_parts))
if group_cols:
- lines.append("GROUP BY " + ", ".join(group_cols))
+ lines.append("GROUP BY " + ", ".join(_quote(c) for c in group_cols))
if params.order_by:
- lines.append(f"ORDER BY {params.order_by} {params.order_dir}")
+ lines.append(f"ORDER BY {_quote(params.order_by)} {params.order_dir}")
elif params.graph_type == "timeseries":
lines.append("ORDER BY bucket")
if params.limit is not None:
@@ -413,8 +421,9 @@ def create_app(db_file: str | Path | None = None) -> Flask:
cached = _cache_get(key)
if cached is not None:
return jsonify(cached)
+ qcol = _quote(column)
rows = con.execute(
- f"SELECT DISTINCT {column} FROM \"{table}\" WHERE CAST({column} AS
VARCHAR) ILIKE '%' || ? || '%' LIMIT 20",
+ f"SELECT DISTINCT {qcol} FROM \"{table}\" WHERE CAST({qcol} AS VARC
HAR) ILIKE '%' || ? || '%' LIMIT 20",
[substr],
).fetchall()
values = [r[0] for r in rows]
@@ -560,10 +569,11 @@ def create_app(db_file: str | Path | None = None) -> Flask
:
params.x_axis or params.time_column
):
axis = params.x_axis or params.time_column
+ assert axis is not None
row = cast(
tuple[datetime | None, datetime | None],
con.execute(
- f'SELECT min({axis}), max({axis}) FROM "{params.table}"'(scubaduck) root@eb913c2329bf:/workspace/scubaduck# git diff scubaduck/server.py
| sed -n '160,320p'
- f'SELECT min({axis}), max({axis}) FROM "{params.table}"'
+ f'SELECT min({_quote(axis)}), max({_quote(axis)}) FROM "{pa
rams.table}"'
).fetchall()[0],
)
mn, mx = row
(scubaduck) root@eb913c2329bf:/workspace/scubaduck#I'm reviewing the differences to ensure the tests are correctly updated.
(scubaduck) root@eb913c2329bf:/workspace/scubaduck# git diff tests/test_server.p
y | tail -n 20
assert "start" not in data and "end" not in data
assert len(data["rows"]) == 4
+
+
+def test_reserved_word_column() -> None:
+ app = server.create_app("TEST")
+ client = app.test_client()
+ payload = {
+ "table": "extra",
+ "columns": ["ts", "desc"],
+ "order_by": "ts",
+ "time_column": "",
+ }
+ rv = client.post(
+ "/api/query", data=json.dumps(payload), content_type="application/json"
+ )
+ data = rv.get_json()
+ assert rv.status_code == 200
+ assert len(data["rows"]) == 2
+ assert data["rows"][0][1] == "x"(scubaduck) root@eb913c2329bf:/workspace/scubaduck# git add scubaduck/server.py
tests/test_server.py(scubaduck) root@eb913c2329bf:/workspace/scubaduck# git commit -m "fix: quote co
lumns in SQL" && git status --short
[work 9528136] fix: quote columns in SQL
2 files changed, 50 insertions(+), 22 deletions(-)(scubaduck) root@eb913c2329bf:/workspace/scubaduck#I realized I forgot to run the status after committing; let's check that now.
(scubaduck) root@eb913c2329bf:/workspace/scubaduck# git status --short
(scubaduck) root@eb913c2329bf:/workspace/scubaduck#I’ll gather the relevant lines from server.py, focusing on the new _quote function and referencing.
(scubaduck) root@eb913c2329bf:/workspace/scubaduck# nl -ba scubaduck/server.py |
sed -n '12,30p'
12 import traceback
13 import math
14
15 import duckdb
16 from dateutil import parser as dtparser
17 from flask import Flask, jsonify, request, send_from_directory
18
19
20 def _quote(ident: str) -> str:
21 """Return identifier quoted for SQL."""
22 return f'"{ident.replace('"', '""')}"'
23
24
25 @dataclass
26 class Filter:
27 column: str
28 op: str
29 value: str | int | float | list[str] | None
30
(scubaduck) root@eb913c2329bf:/workspace/scubaduck#I'll grab the line ranges where I made changes to filtering and quoting.
sed -n '220,320p'
220 f"CAST(floor(epoch({xexpr})/{sec}) AS BIGINT)"
221 )
222 select_parts.append(f"{bucket_expr} AS bucket")
223 group_cols = ["bucket"] + group_cols
224 has_agg = bool(group_cols) or params.aggregate is not None
225 if has_agg:
226 select_cols = (
227 group_cols[1:] if params.graph_type == "timeseries" else gro
up_cols
228 )
229 select_parts.extend(_quote(c) for c in select_cols)
230 agg = (params.aggregate or "count").lower()
231
232 def agg_expr(col: str) -> str:
233 expr = _quote(col)
234 ctype = column_types.get(col, "").upper() if column_types el
se ""
235 if "BOOL" in ctype:
236 expr = f"CAST({_quote(col)} AS BIGINT)"
237 if agg.startswith("p"):
238 quant = float(agg[1:]) / 100
239 return f"quantile({expr}, {quant})"
240 if agg == "count distinct":
241 return f"count(DISTINCT {expr})"
242 if agg == "avg" and column_types is not None:
243 if "TIMESTAMP" in ctype or "DATE" in ctype or "TIME" in
ctype:
244 return (
245 "TIMESTAMP 'epoch' + INTERVAL '1 second' * "
246 f"CAST(avg(epoch({_quote(col)})) AS BIGINT)"
247 )
248 return f"{agg}({expr})"
249
250 if agg == "count":
251 select_parts.append("count(*) AS Count")
252 else:
253 for col in params.columns:
254 if col in group_cols:
255 continue
256 select_parts.append(f"{agg_expr(col)} AS {_quote(col)}")
257 if params.show_hits:
258 select_parts.insert(len(group_cols), "count(*) AS Hits")
259 else:
260 select_parts.extend(_quote(c) for c in params.columns)
261
262 if has_agg and params.derived_columns:
263 inner_params = replace(
264 params,
265 derived_columns={},
266 order_by=None,
267 limit=None,
268 )
269 inner_sql = build_query(inner_params, column_types)
270 outer_select = ["t.*"] + [
271 f"{expr} AS {name}" for name, expr in params.derived_columns
.items()
272 ]
273 indented_inner = "\n".join(" " + line for line in inner_sql.s
plitlines())
274 lines = [
275 f"SELECT {', '.join(outer_select)}",
276 "FROM (",
277 indented_inner,
278 ") t",
279 ]
280 if params.order_by:
281 lines.append(f"ORDER BY {_quote(params.order_by)} {params.or
der_dir}")
282 elif params.graph_type == "timeseries":
283 lines.append("ORDER BY bucket")
284 if params.limit is not None:
285 lines.append(f"LIMIT {params.limit}")
286 return "\n".join(lines)
287
288 for name, expr in params.derived_columns.items():
289 select_parts.append(f"{expr} AS {name}")
290 select_clause = ", ".join(select_parts) if select_parts else "*"
291 lines = [f"SELECT {select_clause}", f'FROM "{params.table}"']
292 where_parts: list[str] = []
293 if params.time_column:
294 time_expr = _time_expr(params.time_column, column_types, params.
time_unit)
295 else:
296 time_expr = None
297 if time_expr and params.start:
298 where_parts.append(f"{time_expr} >= '{params.start}'")
299 if time_expr and params.end:
300 where_parts.append(f"{time_expr} <= '{params.end}'")
301 for f in params.filters:
302 op = f.op
303 if op in {"empty", "!empty"}:
304 val = "''"
305 else:
306 if f.value is None:
307 continue
308 if isinstance(f.value, list):
309 if not f.value:
310 continue
311 if op == "=":
312 qcol = _quote(f.column)
313 vals = " OR ".join(f"{qcol} = '{v}'" for v in f.valu
e)
314 where_parts.append(f"({vals})")
315 continue
316 val = f"'{f.value}'" if isinstance(f.value, str) else str(f.
value)
317
318 qcol = _quote(f.column)
319 if op == "contains":
320 where_parts.append(f"{qcol} ILIKE '%' || {val} || '%'")(scubaduck) root@eb913c2329bf:/workspace/scubaduck# nl -ba scubaduck/server.py |
sed -n '320,380p'
320 where_parts.append(f"{qcol} ILIKE '%' || {val} || '%'")
321 elif op == "!contains":
322 where_parts.append(f"{qcol} NOT ILIKE '%' || {val} || '%'")
323 elif op == "empty":
324 where_parts.append(f"{qcol} = {val}")
325 elif op == "!empty":
326 where_parts.append(f"{qcol} != {val}")
327 else:
328 where_parts.append(f"{qcol} {op} {val}")
329 if where_parts:
330 lines.append("WHERE " + " AND ".join(where_parts))
331 if group_cols:
332 lines.append("GROUP BY " + ", ".join(_quote(c) for c in group_co
ls))
333 if params.order_by:
334 lines.append(f"ORDER BY {_quote(params.order_by)} {params.order_
dir}")
335 elif params.graph_type == "timeseries":
336 lines.append("ORDER BY bucket")
337 if params.limit is not None:
338 lines.append(f"LIMIT {params.limit}")
339 return "\n".join(lines)
340
341
342 def create_app(db_file: str | Path | None = None) -> Flask:
343 app = Flask(__name__, static_folder="static")
344 if db_file is None:
345 env_db = os.environ.get("SCUBADUCK_DB")
346 if env_db:
347 db_file = env_db
348 if isinstance(db_file, str) and db_file.upper() == "TEST":
349 con = _create_test_database()
350 else:
351 db_path = Path(db_file or Path(__file__).with_name("sample.csv")
).resolve()
352 con = _load_database(db_path)
353 tables = [r[0] for r in con.execute("SHOW TABLES").fetchall()]
354 if not tables:
355 raise ValueError("No tables found in database")
356 default_table = tables[0]
357 columns_cache: Dict[str, Dict[str, str]] = {}
358
359 def get_columns(table: str) -> Dict[str, str]:
360 if table not in columns_cache:
361 rows = con.execute(f'PRAGMA table_info("{table}")').fetchall
()
362 if not rows:
363 raise ValueError(f"Unknown table: {table}")
364 columns_cache[table] = {r[1]: r[2] for r in rows}
365 return columns_cache[table]
366
367 sample_cache: Dict[Tuple[str, str, str], Tuple[List[str], float]] =
{}
368 CACHE_TTL = 60.0
369 CACHE_LIMIT = 200
370
371 @app.route("/")
372 def index() -> Any: # pyright: ignore[reportUnusedFunction]
373 assert app.static_folder is not None
374 return send_from_directory(app.static_folder, "index.html")
375
376 @app.route("/js/<path:filename>")
377 def js(filename: str) -> Any: # pyright: ignore[reportUnusedFunctio
n]
378 assert app.static_folder is not None
379 folder = Path(app.static_folder) / "js"
380 return send_from_directory(folder, filename)
(scubaduck) root@eb913c2329bf:/workspace/scubaduck#I'll gather the surrounding line ranges for the sample_values quotations.
(scubaduck) root@eb913c2329bf:/workspace/scubaduck# nl -ba scubaduck/server.py |
sed -n '406,436p'
406 oldest = min(sample_cache.items(), key=lambda kv: kv[1][1])[
0]
407 del sample_cache[oldest]
408
409 @app.route("/api/samples")
410 def sample_values() -> Any: # pyright: ignore[reportUnusedFunction]
411 table = request.args.get("table", default_table)
412 column = request.args.get("column")
413 substr = request.args.get("q", "")
414 column_types = get_columns(table)
415 if not column or column not in column_types:
416 return jsonify([])
417 ctype = column_types[column].upper()
418 if "CHAR" not in ctype and "STRING" not in ctype and "VARCHAR" n
ot in ctype:
419 return jsonify([])
420 key = (table, column, substr)
421 cached = _cache_get(key)
422 if cached is not None:
423 return jsonify(cached)
424 qcol = _quote(column)
425 rows = con.execute(
426 f"SELECT DISTINCT {qcol} FROM \"{table}\" WHERE CAST({qcol}
AS VARCHAR) ILIKE '%' || ? || '%' LIMIT 20",
427 [substr],
428 ).fetchall()
429 values = [r[0] for r in rows]
430 _cache_set(key, values)
431 return jsonify(values)
432
433 @app.route("/api/query", methods=["POST"])
434 def query() -> Any: # pyright: ignore[reportUnusedFunction]
435 payload = request.get_json(force=True)
436 try:
(scubaduck) root@eb913c2329bf:/workspace/scubaduck#I need to get the relevant line ranges for the axis quoting changes.
(scubaduck) root@eb913c2329bf:/workspace/scubaduck# nl -ba scubaduck/server.py |
sed -n '560,580p'
560 return (
561 jsonify(
562 {
563 "error": f"Aggregate {agg} cannot be
applied to column {c}",
564 }
565 ),
566 400,
567 )
568 if (params.start is None or params.end is None) and (
569 params.x_axis or params.time_column
570 ):
571 axis = params.x_axis or params.time_column
572 assert axis is not None
573 row = cast(
574 tuple[datetime | None, datetime | None],
575 con.execute(
576 f'SELECT min({_quote(axis)}), max({_quote(axis)}) FR
OM "{params.table}"'
577 ).fetchall()[0],
578 )
579 mn, mx = row
580 divisor = {
(scubaduck) root@eb913c2329bf:/workspace/scubaduck#I'll grab the relevant lines from the test files.
(scubaduck) root@eb913c2329bf:/workspace/scubaduck# nl -ba tests/test_server.py
| sed -n '820,860p'
820 "columns": ["timestamp", "event"],
821 "time_column": "",
822 }
823 rv = client.post(
824 "/api/query", data=json.dumps(payload), content_type="applicatio
n/json"
825 )
826 data = rv.get_json()
827 assert rv.status_code == 200
828 assert "start" not in data and "end" not in data
829 assert len(data["rows"]) == 4
830
831
832 def test_reserved_word_column() -> None:
833 app = server.create_app("TEST")
834 client = app.test_client()
835 payload = {
836 "table": "extra",
837 "columns": ["ts", "desc"],
838 "order_by": "ts",
839 "time_column": "",
840 }
841 rv = client.post(
842 "/api/query", data=json.dumps(payload), content_type="applicatio
n/json"
843 )
844 data = rv.get_json()
845 assert rv.status_code == 200
846 assert len(data["rows"]) == 2
847 assert data["rows"][0][1] == "x"
(scubaduck) root@eb913c2329bf:/workspace/scubaduck#Time to prepare the PR summary. I'll go ahead and run container.make_pr now.