Codex Task Logs

Task ID: task_e_682fe5e44c8483239f5a3dca9d1fff81

Environment setup
Configuring language runtimes... + rm -- /tmp/oZf83H-setup_script.sh + source ./codex_setup.sh ++ uv sync --frozen Using CPython 3.12.3 interpreter at: /usr/bin/python3.12 Creating virtual environment at: .venv Building scubaduck @ file:///workspace/scubaduck Downloading duckdb (19.3MiB) Downloading pyright (5.3MiB) Downloading playwright (43.1MiB) Downloading ruff (11.0MiB) Downloading ruff Built scubaduck @ file:///workspace/scubaduck Downloading duckdb Downloading playwright Downloading pyright Prepared 33 packages in 1.62s Installed 33 packages in 79ms + blinker==1.9.0 + certifi==2025.4.26 + charset-normalizer==3.4.2 + click==8.2.0 + duckdb==1.2.2 + execnet==2.1.1 + flask==3.1.1 + greenlet==3.2.2 + idna==3.10 + iniconfig==2.1.0 + itsdangerous==2.2.0 + jinja2==3.1.6 + markupsafe==3.0.2 + nodeenv==1.9.1 + packaging==25.0 + playwright==1.52.0 + pluggy==1.6.0 + pyee==13.0.0 + pyright==1.1.400 + pytest==8.3.5 + pytest-base-url==2.1.0 + pytest-playwright==0.7.0 + pytest-xdist==3.6.1 + python-dateutil==2.9.0.post0 + python-slugify==8.0.4 + requests==2.32.3 + ruff==0.11.10 + scubaduck==0.1.0 (from file:///workspace/scubaduck) + six==1.17.0 + text-unidecode==1.3 + typing-extensions==4.13.2 + urllib3==2.4.0 + werkzeug==3.1.3 ++ source .venv/bin/activate +++ '[' -z '' ']' +++ '[' -n x ']' +++ SCRIPT_PATH=.venv/bin/activate +++ '[' .venv/bin/activate = /tmp/oZf83H-setup_script.sh ']' +++ deactivate nondestructive +++ unset -f pydoc +++ '[' -z '' ']' +++ '[' -z '' ']' +++ hash -r +++ '[' -z '' ']' +++ unset VIRTUAL_ENV +++ unset VIRTUAL_ENV_PROMPT +++ '[' '!' nondestructive = nondestructive ']' +++ VIRTUAL_ENV=/workspace/scubaduck/.venv +++ '[' linux-gnu = cygwin ']' +++ '[' linux-gnu = msys ']' +++ export VIRTUAL_ENV +++ '[' -z '' ']' +++ unset SCRIPT_PATH +++ _OLD_VIRTUAL_PATH=/root/.cargo/bin:/root/.rbenv/shims:/root/.rbenv/bin:/root/.rbenv/shims:/root/.local/share/swiftly/bin:/root/.bun/bin:/root/.nvm/versions/node/v22.16.0/bin:/root/.pyenv/shims:3441PYENV_ROOT/shims:/root/.pyenv/bin:/usr/local/go/bin:/root/go/bin:/root/.rbenv/bin:/root/.rbenv/shims:/root/.bun/bin:/root/.local/bin:/root/.pyenv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +++ PATH=/workspace/scubaduck/.venv/bin:/root/.cargo/bin:/root/.rbenv/shims:/root/.rbenv/bin:/root/.rbenv/shims:/root/.local/share/swiftly/bin:/root/.bun/bin:/root/.nvm/versions/node/v22.16.0/bin:/root/.pyenv/shims:3441PYENV_ROOT/shims:/root/.pyenv/bin:/usr/local/go/bin:/root/go/bin:/root/.rbenv/bin:/root/.rbenv/shims:/root/.bun/bin:/root/.local/bin:/root/.pyenv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +++ export PATH +++ '[' xscubaduck '!=' x ']' +++ VIRTUAL_ENV_PROMPT=scubaduck +++ export VIRTUAL_ENV_PROMPT +++ '[' -z '' ']' +++ '[' -z '' ']' +++ _OLD_VIRTUAL_PS1= +++ PS1='(scubaduck) ' +++ export PS1 +++ alias pydoc +++ true +++ hash -r ++ python -c 'import os; import duckdb; con = duckdb.connect(); con.execute(f"SET http_proxy = '\''{os.getenv("HTTP_PROXY")}'\''"); con.execute("INSTALL '\''sqlite'\'';")' ++ playwright install chromium Downloading Chromium 136.0.7103.25 (playwright build v1169) from https://cdn.playwright.dev/dbazure/download/playwright/builds/chromium/1169/chromium-linux.zip 167.7 MiB [] 0% 0.0s167.7 MiB [] 0% 51.5s167.7 MiB [] 0% 56.9s167.7 MiB [] 0% 42.8s167.7 MiB [] 0% 32.8s167.7 MiB [] 0% 30.0s167.7 MiB [] 0% 19.4s167.7 MiB [] 0% 14.0s167.7 MiB [] 1% 10.3s167.7 MiB [] 2% 7.6s167.7 MiB [] 3% 5.0s167.7 MiB [] 4% 4.1s167.7 MiB [] 5% 3.8s167.7 MiB [] 5% 3.6s167.7 MiB [] 7% 3.0s167.7 MiB [] 8% 2.6s167.7 MiB [] 9% 2.5s167.7 MiB [] 10% 2.4s167.7 MiB [] 12% 2.2s167.7 MiB [] 13% 2.3s167.7 MiB [] 13% 2.4s167.7 MiB [] 14% 2.3s167.7 MiB [] 15% 2.2s167.7 MiB [] 17% 2.0s167.7 MiB [] 18% 2.0s167.7 MiB [] 19% 1.9s167.7 MiB [] 20% 1.9s167.7 MiB [] 21% 1.8s167.7 MiB [] 22% 1.7s167.7 MiB [] 24% 1.7s167.7 MiB [] 24% 1.6s167.7 MiB [] 26% 1.6s167.7 MiB [] 27% 1.5s167.7 MiB [] 29% 1.5s167.7 MiB [] 30% 1.4s167.7 MiB [] 31% 1.4s167.7 MiB [] 32% 1.3s167.7 MiB [] 33% 1.3s167.7 MiB [] 34% 1.3s167.7 MiB [] 36% 1.2s167.7 MiB [] 37% 1.2s167.7 MiB [] 38% 1.2s167.7 MiB [] 39% 1.1s167.7 MiB [] 40% 1.1s167.7 MiB [] 42% 1.1s167.7 MiB [] 43% 1.0s167.7 MiB [] 44% 1.0s167.7 MiB [] 46% 1.0s167.7 MiB [] 47% 1.0s167.7 MiB [] 48% 0.9s167.7 MiB [] 49% 0.9s167.7 MiB [] 50% 0.9s167.7 MiB [] 51% 0.9s167.7 MiB [] 52% 0.8s167.7 MiB [] 53% 0.8s167.7 MiB [] 55% 0.8s167.7 MiB [] 56% 0.8s167.7 MiB [] 58% 0.7s167.7 MiB [] 59% 0.7s167.7 MiB [] 60% 0.7s167.7 MiB [] 61% 0.7s167.7 MiB [] 62% 0.6s167.7 MiB [] 64% 0.6s167.7 MiB [] 65% 0.6s167.7 MiB [] 66% 0.6s167.7 MiB [] 67% 0.6s167.7 MiB [] 68% 0.5s167.7 MiB [] 69% 0.5s167.7 MiB [] 71% 0.5s167.7 MiB [] 72% 0.5s167.7 MiB [] 74% 0.5s167.7 MiB [] 75% 0.4s167.7 MiB [] 77% 0.4s167.7 MiB [] 78% 0.4s167.7 MiB [] 79% 0.3s167.7 MiB [] 81% 0.3s167.7 MiB [] 83% 0.3s167.7 MiB [] 86% 0.2s167.7 MiB [] 88% 0.2s167.7 MiB [] 89% 0.2s167.7 MiB [] 90% 0.2s167.7 MiB [] 92% 0.1s167.7 MiB [] 93% 0.1s167.7 MiB [] 94% 0.1s167.7 MiB [] 95% 0.1s167.7 MiB [] 96% 0.1s167.7 MiB [] 98% 0.0s167.7 MiB [] 99% 0.0s167.7 MiB [] 100% 0.0s Chromium 136.0.7103.25 (playwright build v1169) downloaded to /root/.cache/ms-playwright/chromium-1169 Downloading FFMPEG playwright build v1011 from https://cdn.playwright.dev/dbazure/download/playwright/builds/ffmpeg/1011/ffmpeg-linux.zip 2.3 MiB [] 0% 0.0s2.3 MiB [] 4% 0.3s2.3 MiB [] 5% 0.7s2.3 MiB [] 12% 0.4s2.3 MiB [] 17% 0.4s2.3 MiB [] 33% 0.2s2.3 MiB [] 45% 0.1s2.3 MiB [] 64% 0.1s2.3 MiB [] 91% 0.0s2.3 MiB [] 100% 0.0s FFMPEG playwright build v1011 downloaded to /root/.cache/ms-playwright/ffmpeg-1011 Downloading Chromium Headless Shell 136.0.7103.25 (playwright build v1169) from https://cdn.playwright.dev/dbazure/download/playwright/builds/chromium/1169/chromium-headless-shell-linux.zip 101.4 MiB [] 0% 0.0s101.4 MiB [] 0% 1.7s101.4 MiB [] 2% 1.0s101.4 MiB [] 4% 1.0s101.4 MiB [] 6% 0.9s101.4 MiB [] 8% 0.8s101.4 MiB [] 10% 0.9s101.4 MiB [] 12% 0.8s101.4 MiB [] 14% 0.7s101.4 MiB [] 17% 0.7s101.4 MiB [] 20% 0.7s101.4 MiB [] 21% 0.7s101.4 MiB [] 23% 0.6s101.4 MiB [] 24% 0.6s101.4 MiB [] 26% 0.7s101.4 MiB [] 27% 0.7s101.4 MiB [] 30% 0.7s101.4 MiB [] 33% 0.6s101.4 MiB [] 34% 0.6s101.4 MiB [] 36% 0.6s101.4 MiB [] 38% 0.6s101.4 MiB [] 40% 0.6s101.4 MiB [] 42% 0.5s101.4 MiB [] 44% 0.5s101.4 MiB [] 46% 0.5s101.4 MiB [] 49% 0.5s101.4 MiB [] 50% 0.4s101.4 MiB [] 52% 0.4s101.4 MiB [] 55% 0.4s101.4 MiB [] 56% 0.4s101.4 MiB [] 58% 0.4s101.4 MiB [] 61% 0.4s101.4 MiB [] 62% 0.3s101.4 MiB [] 65% 0.3s101.4 MiB [] 67% 0.3s101.4 MiB [] 69% 0.3s101.4 MiB [] 71% 0.3s101.4 MiB [] 73% 0.2s101.4 MiB [] 74% 0.2s101.4 MiB [] 75% 0.2s101.4 MiB [] 76% 0.2s101.4 MiB [] 77% 0.2s101.4 MiB [] 78% 0.2s101.4 MiB [] 80% 0.2s101.4 MiB [] 82% 0.2s101.4 MiB [] 84% 0.2s101.4 MiB [] 87% 0.1s101.4 MiB [] 91% 0.1s101.4 MiB [] 93% 0.1s101.4 MiB [] 99% 0.0s101.4 MiB [] 100% 0.0s Chromium Headless Shell 136.0.7103.25 (playwright build v1169) downloaded to /root/.cache/ms-playwright/chromium_headless_shell-1169 ++ echo 'source .venv/bin/activate'
Internet turned off
Bug: Go to Table view, add a Group By, and then Order By Samples. It says "Unknown column: Samples". The Samples column is actually the computed "hits", i.e., COUNT(1). Add a test for this and fix it. I think there might be some overzealous server validation, as we never hit the query.

I’ll open AGENTS.md in the repo root to check for relevant instructions to follow.

shell
bash: .venv/bin/activate: No such file or directory root@5b69743466e4:/workspace# ls scubaduck
root@5b69743466e4:/workspace# cd scubaduck root@5b69743466e4:/workspace/scubaduck# ls AGENTS.md codex_setup.sh scubaduck tests README.md pyproject.toml stubs uv.lock
root@5b69743466e4:/workspace/scubaduck# cat AGENTS.md - Use "source .venv/bin/activate" to activate the virtual environment before running any commands - ONLY use non-interactive commands like cat, sed, apply_patch to do edits. Do NOT use interactive editors. - Do NOT attempt to install packages. Only the packages specified in pyproject.toml are available. You cannot add new packages. If you desperately want another package, make a note of it in the final PR description. - Use conventional commits to format PR title - There are no nested AGENTS.md files, this is the only agents file - When using Playwright, ONLY use chromium browser. This browser is already installed. - Use "ruff check" to check lint, "ruff format" to autoformat files and "pyright" to typecheck. - When writing the PR description, include the original prompt supplied by the user. - When you add functionality to the server, add server tests to tests/test_server.py. When you add features to the frontend, add frontend tests to tests/test_web.py

I'll activate the Python virtual environment and ensure proper repo setup before proceeding.

shell
root@5b69743466e4:/workspace/scubaduck# source .venv/bin/activate (scubaduck) root@5b69743466e4:/workspace/scubaduck#

I'll take a look at the repository to see what’s inside.

shell
(scubaduck) root@5b69743466e4:/workspace/scubaduck# ls AGENTS.md codex_setup.sh scubaduck tests README.md pyproject.toml stubs uv.lock (scubaduck) root@5b69743466e4:/workspace/scubaduck#

I'll start by checking out the README to understand the project better.

shell
(scubaduck) root@5b69743466e4:/workspace/scubaduck# cat README.md I really like Scuba (Meta's internal real-time database system). The distributed , real-time database part of Scuba is quite difficult (and expensive) to replica te, but I also really like Scuba's UI for doing queries, and I have found myself wishing that I have access to it even for "small" databases, e.g., I have a sql ite dataset I want to explore. Pivotal ideas: * Time series by default. In the dedicated "time series" view, there are many fe atures specifically oriented towards working towards tables that represent event s that occurred over time: the start, end, compare, aggregate and granularity fi elds all specially privilege the timestamp field. In fact, you can't log events to Scuba's backing data store without a timestamp, they always come with one. (S cuba also supports other views that don't presuppose a time series, but the time series is the most beloved and well used view.) This is in contrast to typical software which tries to generalize to arbitrary data first, with time series bei ng added on later. * It's all about exploration. Scuba is predicated on the idea that you don't kno w what you're looking for, that you are going to spend time tweaking queries and changing filters/grouping as part of an investigation to figure out why a syste m behaves the way it is. So the filters/comparisons/groupings you want to edit a re always visible on the left sidebar, with the expectation that you're going to tweak the query to look at something else. Similarly, all the parameters of you r query get saved into your URL, so your browser history can double up as a quer y history / you can easily share a query with someone else. This is contrast to typical software which is often oriented to making pretty dashboards and reports . (This function is important too, but it's not what I want in exploration mode! ) * You can fix data problems in the query editor. It's pretty common to have mess ed up and ended up with a database that doesn't have exactly the columns you nee d, or some columns that are corrupted in some way. Scuba has pretty robust suppo rt for defining custom columns with arbitrary SQL functions, grouping over them as if they were native functions, and doing so with minimal runtime cost (Scuba aims to turn around your query in milliseconds!) Having to go and run a huge dat a pipeline to fix your data is a big impediment to exploration; quick and easy c ustom columns means you can patch over problems when you're investigating and fi x them for real later. We're going to build a exploratory data analysis tool like Scuba for time series database (i.e., a database with a mandatory timestamp representing the time an event occurred). We'll use DuckDB as the underlying SQL engine served from a Py thon server, and render the GUI/results as a webpage with vanilla HTML and JS. W e'll use choices.js to support token inputs. We define a token input to mean a text input element where as you type a dropdown displays with valid values, and if you select one or press enter, the selection turns into a token/chip that can only be deleted as one unit. To start, we are going to support one views: samples. The samples view only all ows you to view individual samples from the database, subject to a filter. Our m ain UI concept is that there is a left sidebar that is the query editor, and the right side that shows the view. The sidebar is always visible and defaults to the query parameters of the current view. After you make changes to the query, clicking the "Dive" button updates the view. The URL of the page encodes all of the values of the query (and gets updated when you Dive), so the browser's back button lets you view previous queries. The query editor's job is to generate a SQL query, which then is applied on the database, and then the result visualized according to the view. Here are the settings you can apply to the query. The help text should show up w hen you mouse over the field name: * Start/End - Help text: "Sets the start/end of the time range to query. Can be any kind of datetime string. For example: 'April 23, 2014' or 'yesterday'." The UI for this selector supports both relative selections (now, -1 hour, -3 hours, -12 hours, -1 day, -3 days, -1 week, -1 fortnight, -30 days, -90 days, -1 month, -1 year) as well as specifying an absolute date. The way this field is rendere d is there is a free form text box, a drop down arrow (for the relative selector s), and then a calendar button (for date selection). * Order By - Help text: "Choose a column to sort results by." There is an ASC/D ESC toggle next to it. * Limit - Help text: "Choose the maximum number of results to show in the chart after any aggregations have been applied. For example, a limit of 10 will show no more than 10 rows for a table, etc." * Filters - You can create as many filters as you want. You can either write a f ilter using a UI or manual SQL. In the UI, filter consists of a column name, a r elation (e.g., =, !=, <, >) and then a text field. The text field is a token inp ut. It accepts multiple tokens for = relation, in which case we match using an O R for all options. There is also a "Columns" tab which lets you view all fields in the table, organ ized by their type. You can also define derived columns, by specifying a column name and SQL expression. Derived columns can be used for all parts of the UI, in cluding filters/group by/etc. Columns have checkboxes indicating if we should SE LECT them or not. Each selected column shows up in the graph. There is an All/N one link which can be used to select/deselect all checkboxes. The query UI constructs a SQL query that intuitively has this form: ``` SELECT column, column, ..., FROM table WHERE time >= min-timestamp AND time <= max-timestamp [AND condition ...] ORDER BY aggregate(column) LIMIT number ``` You should write tests for the server backend, demonstrating that at specific qu ery values we get back the correct rows of data. ## Running the server Activate the virtual environment and run the Flask development server: ```bash flask --app scubaduck.server run --debug ``` By default the server loads `sample.csv`. Set the `SCUBADUCK_DB` environment variable to point at a different database file (CSV, SQLite or DuckDB) if you want to use another dataset. The special value `TEST` starts the server with a small in-memory SQLite dataset used by the automated tests. If the file does not exist, the server will raise a `FileNotFoundError` during startup. (scubaduck) root@5b69743466e4:/workspace/scubaduck#

I'll check the repository to understand the server-side issue with the "Samples" column.

shell
(scubaduck) root@5b69743466e4:/workspace/scubaduck# ls scubaduck __init__.py sample.csv server.py static (scubaduck) root@5b69743466e4:/workspace/scubaduck#

I’ll take a look at server.py to see what’s going on there.

shell
(scubaduck) root@5b69743466e4:/workspace/scubaduck# sed -n '1,160p' scubaduck/se rver.py from __future__ import annotations from dataclasses import dataclass, field, replace from typing import Any, Dict, List, Tuple, cast import re from datetime import datetime, timedelta, timezone import time from pathlib import Path import os import traceback import math import duckdb from dateutil import parser as dtparser from dateutil.relativedelta import relativedelta from flask import Flask, jsonify, request, send_from_directory def _quote(ident: str) -> str: """Return identifier quoted for SQL.""" return f'"{ident.replace('"', '""')}"' @dataclass class Filter: column: str op: str value: str | int | float | list[str] | None @dataclass class QueryParams: start: str | None = None end: str | None = None order_by: str | None = None order_dir: str = "ASC" limit: int | None = None columns: list[str] = field(default_factory=lambda: []) filters: list[Filter] = field(default_factory=lambda: []) derived_columns: dict[str, str] = field(default_factory=lambda: {}) graph_type: str = "samples" group_by: list[str] = field(default_factory=lambda: []) aggregate: str | None = None show_hits: bool = False x_axis: str | None = None granularity: str = "Auto" fill: str = "0" table: str = "events" time_column: str | None = "timestamp" time_unit: str = "s" def _load_database(path: Path) -> duckdb.DuckDBPyConnection: if not path.exists(): raise FileNotFoundError(path) ext = path.suffix.lower() if ext == ".csv": con = duckdb.connect() con.execute( f"CREATE TABLE events AS SELECT * FROM read_csv_auto('{path.as_posix ()}')" ) elif ext in {".db", ".sqlite"}: con = duckdb.connect() con.execute("LOAD sqlite") con.execute(f"ATTACH '{path.as_posix()}' AS db (TYPE SQLITE)") tables = [ r[0] for r in con.execute( "SELECT name FROM sqlite_master WHERE type='table'" ).fetchall() ] for t in tables: con.execute(f'CREATE VIEW "{t}" AS SELECT * FROM db."{t}"') else: con = duckdb.connect(path) return con def _create_test_database() -> duckdb.DuckDBPyConnection: """Return a DuckDB connection with a small multi-table dataset.""" con = duckdb.connect() con.execute( "CREATE TABLE events (id INTEGER PRIMARY KEY, ts INTEGER, val REAL, name TEXT, flag BOOLEAN)" ) con.execute("INSERT INTO events VALUES (1, 1704067200, 1.5, 'alice', 1)") con.execute("INSERT INTO events VALUES (2, 1704070800, 2.0, 'bob', 0)") con.execute('CREATE TABLE extra (ts INTEGER, "desc" TEXT, num INTEGER)') con.execute("INSERT INTO extra VALUES (1704067200, 'x', 1)") con.execute("INSERT INTO extra VALUES (1704070800, 'y', 2)") return con _REL_RE = re.compile( r"([+-]?\d+(?:\.\d*)?)\s*(hour|hours|day|days|week|weeks|fortnight|fortnight s|month|months|year|years)", re.IGNORECASE, ) def parse_time(val: str | None) -> str | None: """Parse an absolute or relative time string into ``YYYY-MM-DD HH:MM:SS``."" " if val is None or val == "": return None s = val.strip() if s.lower() == "now": dt = datetime.now(timezone.utc) return dt.replace(microsecond=0).strftime("%Y-%m-%d %H:%M:%S") m = _REL_RE.fullmatch(s) if m: qty = float(m.group(1)) unit = m.group(2).lower() now = datetime.now(timezone.utc) dt: datetime if unit.startswith("hour"): dt = now + timedelta(hours=qty) elif unit.startswith("day"): dt = now + timedelta(days=qty) elif unit.startswith("week"): dt = now + timedelta(weeks=qty) elif unit.startswith("fortnight"): dt = now + timedelta(weeks=2 * qty) elif unit.startswith("month"): if qty.is_integer(): dt = now + relativedelta(months=int(qty)) else: dt = now + timedelta(days=30 * qty) elif unit.startswith("year"): if qty.is_integer(): dt = now + relativedelta(years=int(qty)) else: dt = now + timedelta(days=365 * qty) else: # pragma: no cover - defensive raise ValueError(f"Unsupported unit: {unit}") return dt.replace(microsecond=0).strftime("%Y-%m-%d %H:%M:%S") dt = dtparser.parse(s) return dt.replace(microsecond=0, tzinfo=None).strftime("%Y-%m-%d %H:%M:%S") def _numeric_to_datetime(value: int | float, unit: str) -> datetime: """Convert a numeric timestamp ``value`` with unit ``unit`` to ``datetime``. Heuristically fall back to seconds when the converted value is before 1990 but the seconds interpretation is in a reasonable range. This handles integer columns stored in seconds even when ``unit`` is mistakenly set to a finer granularity. """ divisor = { "s": 1, "ms": 1000, "us": 1_000_000, "ns": 1_000_000_000, }.get(unit, 1) dt = datetime.fromtimestamp(int(value) / divisor, tz=timezone.utc) if unit != "s" and dt.year < 1990:
(scubaduck) root@5b69743466e4:/workspace/scubaduck# sed -n '160,320p' scubaduck/ server.py if unit != "s" and dt.year < 1990: alt = datetime.fromtimestamp(int(value), tz=timezone.utc) if alt.year >= 1990: dt = alt return dt def _suggest_time_unit(value: int | float, given: str) -> str | None: """Return a plausible time unit for ``value`` not equal to ``given``.""" for unit in ("s", "ms", "us", "ns"): if unit == given: continue try: dt = _numeric_to_datetime(value, unit) except Exception: continue if 1990 <= dt.year <= 2500: return unit return None def _granularity_seconds(granularity: str, start: str | None, end: str | None) - > int: gran = granularity.lower() mapping = { "1 second": 1, "5 seconds": 5, "10 seconds": 10, "30 seconds": 30, "1 minute": 60, "4 minutes": 240, "5 minutes": 300, "10 minutes": 600, "15 minutes": 900, "30 minutes": 1800, "1 hour": 3600, "3 hours": 10800, "6 hours": 21600, "1 day": 86400, "1 week": 604800, "30 days": 2592000, } if gran in mapping: return mapping[gran] if gran in {"auto", "fine"} and start and end: try: s = dtparser.parse(start) e = dtparser.parse(end) except Exception: return 3600 total = max((e - s).total_seconds(), 1) buckets = 100 if gran == "auto" else 500 return max(int(total // buckets), 1) return 3600 def _time_expr(col: str, column_types: Dict[str, str] | None, unit: str) -> str: """Return SQL expression for column interpreted as timestamp.""" qcol = _quote(col) if column_types is None: return qcol ctype = column_types.get(col, "").upper() if not any(t in ctype for t in ["TIMESTAMP", "DATE", "TIME"]): if any( t in ctype for t in [ "INT", "DECIMAL", "REAL", "DOUBLE", "FLOAT", "NUMERIC", "HUGEINT", ] ): if unit == "ns": # Use nanosecond helper unless column cannot represent such larg e values if "INT" in ctype and "BIGINT" not in ctype and "HUGEINT" not in ctype: unit = "s" else: expr = f"CAST({qcol} AS BIGINT)" return f"make_timestamp_ns({expr})" if ( unit != "s" and "INT" in ctype and "BIGINT" not in ctype and "HUGEINT" not in ctype ): # 32-bit integers cannot store sub-second precision for modern d ates unit = "s" multiplier = { "s": 1_000_000, "ms": 1_000, "us": 1, }.get(unit, 1_000_000) base = f"CAST({qcol} AS BIGINT)" expr = f"CAST({base} * {multiplier} AS BIGINT)" if multiplier != 1 e lse base return f"make_timestamp({expr})" return qcol def build_query(params: QueryParams, column_types: Dict[str, str] | None = None) -> str: select_parts: list[str] = [] group_cols = params.group_by[:] selected_for_order = set(params.columns) | set(params.derived_columns.keys() ) if params.graph_type == "timeseries": sec = _granularity_seconds(params.granularity, params.start, params.end) x_axis = params.x_axis or params.time_column if x_axis is None: raise ValueError("x_axis required for timeseries") xexpr = _time_expr(x_axis, column_types, params.time_unit) if params.start: bucket_expr = ( f"TIMESTAMP '{params.start}' + INTERVAL '{sec} second' * " f"CAST(floor((epoch({xexpr}) - epoch(TIMESTAMP '{params.start}') )/{sec}) AS BIGINT)" ) else: bucket_expr = ( f"TIMESTAMP 'epoch' + INTERVAL '{sec} second' * " f"CAST(floor(epoch({xexpr})/{sec}) AS BIGINT)" ) select_parts.append(f"{bucket_expr} AS bucket") group_cols = ["bucket"] + group_cols selected_for_order.add("bucket") has_agg = bool(group_cols) or params.aggregate is not None if has_agg: select_cols = ( group_cols[1:] if params.graph_type == "timeseries" else group_cols ) select_parts.extend(_quote(c) for c in select_cols) agg = (params.aggregate or "count").lower() selected_for_order.update(group_cols) def agg_expr(col: str) -> str: expr = _quote(col) ctype = column_types.get(col, "").upper() if column_types else "" if "BOOL" in ctype: expr = f"CAST({_quote(col)} AS BIGINT)" if agg.startswith("p"): quant = float(agg[1:]) / 100 return f"quantile({expr}, {quant})" if agg == "count distinct": return f"count(DISTINCT {expr})" if agg == "avg" and column_types is not None: if "TIMESTAMP" in ctype or "DATE" in ctype or "TIME" in ctype: return ( "TIMESTAMP 'epoch' + INTERVAL '1 second' * " f"CAST(avg(epoch({_quote(col)})) AS BIGINT)" ) return f"{agg}({expr})" if agg == "count": if params.graph_type != "table": select_parts.append("count(*) AS Count") selected_for_order.add("Count") else: for col in params.columns: if col in group_cols: continue
(scubaduck) root@5b69743466e4:/workspace/scubaduck# sed -n '320,640p' scubaduck/ server.py continue select_parts.append(f"{agg_expr(col)} AS {_quote(col)}") selected_for_order.add(col) select_parts.insert(len(group_cols), "count(*) AS Hits") selected_for_order.add("Hits") else: select_parts.extend(_quote(c) for c in params.columns) selected_for_order.update(params.columns) order_by = params.order_by if order_by == "Samples": order_by = "Hits" order_by = order_by if order_by in selected_for_order else None if has_agg and params.derived_columns: inner_params = replace( params, derived_columns={}, order_by=None, limit=None, ) inner_sql = build_query(inner_params, column_types) outer_select = ["t.*"] + [ f"{expr} AS {name}" for name, expr in params.derived_columns.items() ] indented_inner = "\n".join(" " + line for line in inner_sql.splitline s()) lines = [ f"SELECT {', '.join(outer_select)}", "FROM (", indented_inner, ") t", ] if order_by: lines.append(f"ORDER BY {_quote(order_by)} {params.order_dir}") elif params.graph_type == "timeseries": lines.append("ORDER BY bucket") if params.limit is not None: lines.append(f"LIMIT {params.limit}") return "\n".join(lines) for name, expr in params.derived_columns.items(): select_parts.append(f"{expr} AS {name}") selected_for_order.add(name) select_clause = ", ".join(select_parts) if select_parts else "*" lines = [f"SELECT {select_clause}", f'FROM "{params.table}"'] where_parts: list[str] = [] if params.time_column: time_expr = _time_expr(params.time_column, column_types, params.time_uni t) else: time_expr = None if time_expr and params.start: where_parts.append(f"{time_expr} >= '{params.start}'") if time_expr and params.end: where_parts.append(f"{time_expr} <= '{params.end}'") for f in params.filters: op = f.op if op in {"empty", "!empty"}: val = "''" else: if f.value is None: continue if isinstance(f.value, list): if not f.value: continue if op == "=": qcol = _quote(f.column) vals = " OR ".join(f"{qcol} = '{v}'" for v in f.value) where_parts.append(f"({vals})") continue val = f"'{f.value}'" if isinstance(f.value, str) else str(f.value) qcol = _quote(f.column) if op == "contains": where_parts.append(f"{qcol} ILIKE '%' || {val} || '%'") elif op == "!contains": where_parts.append(f"{qcol} NOT ILIKE '%' || {val} || '%'") elif op == "empty": where_parts.append(f"{qcol} = {val}") elif op == "!empty": where_parts.append(f"{qcol} != {val}") else: where_parts.append(f"{qcol} {op} {val}") if where_parts: lines.append("WHERE " + " AND ".join(where_parts)) if group_cols: lines.append("GROUP BY " + ", ".join(_quote(c) for c in group_cols)) if order_by: lines.append(f"ORDER BY {_quote(order_by)} {params.order_dir}") elif params.graph_type == "timeseries": lines.append("ORDER BY bucket") if params.limit is not None: lines.append(f"LIMIT {params.limit}") return "\n".join(lines) def create_app(db_file: str | Path | None = None) -> Flask: app = Flask(__name__, static_folder="static") if db_file is None: env_db = os.environ.get("SCUBADUCK_DB") if env_db: db_file = env_db if isinstance(db_file, str) and db_file.upper() == "TEST": con = _create_test_database() else: db_path = Path(db_file or Path(__file__).with_name("sample.csv")).resolv e() con = _load_database(db_path) tables = [r[0] for r in con.execute("SHOW TABLES").fetchall()] if not tables: raise ValueError("No tables found in database") default_table = tables[0] columns_cache: Dict[str, Dict[str, str]] = {} def get_columns(table: str) -> Dict[str, str]: if table not in columns_cache: rows = con.execute(f'PRAGMA table_info("{table}")').fetchall() if not rows: raise ValueError(f"Unknown table: {table}") columns_cache[table] = {r[1]: r[2] for r in rows} return columns_cache[table] ... columns=payload.get("columns", []), derived_columns=payload.get("derived_columns", {}), graph_type=payload.get("graph_type", "samples"), group_by=payload.get("group_by", []), aggregate=payload.get("aggregate"), show_hits=payload.get("show_hits", False), x_axis=payload.get("x_axis"), granularity=payload.get("granularity", "Auto"), fill=payload.get("fill", "0"), table=payload.get("table", default_table), time_column=payload.get("time_column", "timestamp"), time_unit=payload.get("time_unit", "s"), ) if params.order_by == "Samples": params.order_by = "Hits" for f in payload.get("filters", []): params.filters.append(Filter(f["column"], f["op"], f.get("value"))) if params.table not in tables: return jsonify({"error": "Invalid table"}), 400 column_types = get_columns(params.table) if params.time_column and params.time_column not in column_types: return jsonify({"error": "Invalid time_column"}), 400 if params.time_unit not in {"s", "ms", "us", "ns"}: return jsonify({"error": "Invalid time_unit"}), 400 if params.graph_type not in {"table", "timeseries"} and ( params.group_by or params.aggregate or params.show_hits ): return ( jsonify( { "error": "group_by, aggregate and show_hits are only val id for table or timeseries view" } ), 400, ) valid_cols = set(column_types.keys()) valid_cols.update(params.derived_columns.keys()) valid_cols.add("Hits") if params.graph_type == "timeseries": if params.x_axis is None: params.x_axis = params.time_column if params.x_axis is None or params.x_axis not in valid_cols: return jsonify({"error": "Invalid x_axis"}), 400 ctype = column_types.get(params.x_axis, "").upper() is_time = any(t in ctype for t in ["TIMESTAMP", "DATE", "TIME"]) is_numeric = any( t in ctype for t in [ "INT", "DECIMAL", "REAL", "DOUBLE", "FLOAT", "NUMERIC", "HUGEINT", ] ) if not (is_time or is_numeric): return jsonify({"error": "x_axis must be a time column"}), 400 for col in params.columns: if col not in valid_cols: return jsonify({"error": f"Unknown column: {col}"}), 400 for col in params.group_by: if col not in valid_cols: return jsonify({"error": f"Unknown column: {col}"}), 400 if params.order_by and params.order_by not in valid_cols: return jsonify({"error": f"Unknown column: {params.order_by}"}), 400 if params.group_by or params.graph_type == "timeseries": agg = (params.aggregate or "count").lower() if agg.startswith("p") or agg == "sum": need_numeric = True allow_time = False elif agg == "avg" or agg in {"min", "max"}: need_numeric = False allow_time = True else: need_numeric = False allow_time = False if need_numeric or allow_time: for c in params.columns: if c in params.group_by or c == params.x_axis: continue if c not in column_types: continue ctype = column_types.get(c, "").upper() is_numeric = "BOOL" in ctype or any( t in ctype for t in [ "INT", "DECIMAL", "REAL", "DOUBLE", "FLOAT", "NUMERIC", "HUGEINT", ] ) is_time = "TIMESTAMP" in ctype or "DATE" in ctype or "TIME" in ctype if need_numeric and not is_numeric: return ( jsonify( { "error": f"Aggregate {agg} cannot be applied to column {c}", } ), 400, ) if allow_time and not (is_numeric or is_time): return ( jsonify( { "error": f"Aggregate {agg} cannot be applied to column {c}", } (scubaduck) root@5b69743466e4:/workspace/scubaduck#
(scubaduck) root@5b69743466e4:/workspace/scubaduck# sed -n '640,1280p' scubaduck /server.py } ), 400, ) if (params.start is None or params.end is None) and ( params.x_axis or params.time_column ): axis = params.x_axis or params.time_column assert axis is not None row = cast( tuple[datetime | None, datetime | None], con.execute( f'SELECT min({_quote(axis)}), max({_quote(axis)}) FROM "{par ams.table}"' ).fetchall()[0], ) mn, mx = row if isinstance(mn, (int, float)): try: mn = _numeric_to_datetime(mn, params.time_unit) except Exception: suggestion = _suggest_time_unit(mn, params.time_unit) msg = f"Invalid time value {mn} for column {axis} with time_ unit {params.time_unit}" if suggestion: msg += f"; maybe try time_unit {suggestion}" return jsonify({"error": msg}), 400 if isinstance(mx, (int, float)): try: mx = _numeric_to_datetime(mx, params.time_unit) except Exception: suggestion = _suggest_time_unit(mx, params.time_unit) msg = f"Invalid time value {mx} for column {axis} with time_ unit {params.time_unit}" if suggestion: msg += f"; maybe try time_unit {suggestion}" return jsonify({"error": msg}), 400 if params.start is None and mn is not None: params.start = ( mn.strftime("%Y-%m-%d %H:%M:%S") if not isinstance(mn, str) else mn ) if params.end is None and mx is not None: params.end = ( mx.strftime("%Y-%m-%d %H:%M:%S") if not isinstance(mx, str) else mx ) bucket_size: int | None = None series_limit = params.limit if params.graph_type == "timeseries": bucket_size = _granularity_seconds( params.granularity, params.start if isinstance(params.start, str) else None, params.end if isinstance(params.end, str) else None, ) if ( params.limit is not None and params.start is not None and params.end is not None ): try: start_dt = dtparser.parse(params.start) end_dt = dtparser.parse(params.end) buckets = math.ceil( (end_dt - start_dt).total_seconds() / bucket_size ) if buckets > 1: params.limit *= buckets except Exception: pass sql = build_query(params, column_types) try: rows = con.execute(sql).fetchall() except Exception as exc: tb = traceback.format_exc() print(f"Query failed:\n{sql}\n{tb}") return ( jsonify({"sql": sql, "error": str(exc), "traceback": tb}), 400, ) def _serialize(value: Any) -> Any: if isinstance(value, bytes): return repr(value) return value rows = [[_serialize(v) for v in r] for r in rows] if ( params.graph_type == "timeseries" and params.group_by and series_limit is not None ): key_slice = slice(1, 1 + len(params.group_by)) kept: set[tuple[Any, ...]] = set() filtered: list[list[Any]] = [] for row in rows: key = tuple(row[key_slice]) if key not in kept: if len(kept) >= series_limit: continue kept.add(key) filtered.append(row) rows = filtered result: Dict[str, Any] = {"sql": sql, "rows": rows} if params.start is not None: result["start"] = str(params.start) if params.end is not None: result["end"] = str(params.end) if bucket_size is not None: result["bucket_size"] = bucket_size return jsonify(result) return app app = create_app() if __name__ == "__main__": app.run(debug=True) (scubaduck) root@5b69743466e4:/workspace/scubaduck#

The bug seems to be related to server-side validation, especially with the "Samples" alias.

shell
(scubaduck) root@5b69743466e4:/workspace/scubaduck# grep -n "Samples" -R tests/test_server_timeseries.py:324: "order_by": "Samples", tests/test_server_timeseries.py:346: "order_by": "Samples", scubaduck/static/index.html:193: <div id="header">sample.csv - <select id="tabl e"></select> <select id="graph_type"><option value="samples">Samples</option><op tion value="table">Table</option><option value="timeseries">Time Series</option> </select></div> scubaduck/static/js/view_settings.js:216: samplesOpt.value = 'Samples'; scubaduck/static/js/view_settings.js:217: samplesOpt.textContent = 'Samples'; scubaduck/server.py:330: if order_by == "Samples": scubaduck/server.py:534: if params.order_by == "Samples": .venv/lib64/python3.12/site-packages/pyright/dist/dist/typeshed-fallback/stubs/p ywin32/_win32typing.pyi:2096: def nSamplesPerSec(self) -> int: ... grep: .venv/lib64/python3.12/site-packages/duckdb/duckdb.cpython-312-x86_64-linu x-gnu.so: binary file matches grep: .venv/lib64/python3.12/site-packages/playwright/driver/node: binary file m atches

I see tests referencing Samples, so I'll take a closer look at them.

shell
.venv/lib/python3.12/site-packages/pyright/dist/dist/typeshed-fallback/stubs/pyw in32/_win32typing.pyi:2096: def nSamplesPerSec(self) -> int: ... grep: .venv/lib/python3.12/site-packages/duckdb/duckdb.cpython-312-x86_64-linux- gnu.so: binary file matches grep: .venv/lib/python3.12/site-packages/playwright/driver/node: binary file mat ches (scubaduck) root@5b69743466e4:/workspace/scubaduck# ls tests conftest.py test_server_time.py test_web_graph.py test_multi_table_web.py test_server_timeseries.py test_web_table.py test_server.py test_web.py test_web_timeseries.py test_server_basic.py test_web_basics.py web_utils.py test_server_db_types.py test_web_chips.py test_server_errors.py test_web_columns.py (scubaduck) root@5b69743466e4:/workspace/scubaduck#

I'll check out the tests that mention 'Samples' to see what's going on.

shell
_server_timeseries.py assert rv.status_code == 200 rows = sorted(data["rows"]) assert rows == [[1, 1], [2, 1]] assert all(len(r) == 2 for r in rows) def test_order_by_samples_table() -> None: app = server.app client = app.test_client() payload: dict[str, Any] = { "table": "events", "start": "2024-01-01 00:00:00", "end": "2024-01-03 00:00:00", "graph_type": "table", "order_by": "Samples", "order_dir": "DESC", "limit": 10, "columns": [], "group_by": ["user"], } rv = client.post( "/api/query", data=json.dumps(payload), content_type="application/json" ) data = rv.get_json() assert rv.status_code == 200 assert 'ORDER BY "Hits" DESC' in data["sql"] def test_order_by_samples_timeseries() -> None: app = server.app client = app.test_client() payload: dict[str, Any] = { "table": "events", "start": "2024-01-01 00:00:00", "end": "2024-01-03 00:00:00", "graph_type": "timeseries", "order_by": "Samples", "order_dir": "DESC", "x_axis": "timestamp", "granularity": "1 day", "columns": [], } rv = client.post( "/api/query", data=json.dumps(payload), content_type="application/json" ) data = rv.get_json() assert rv.status_code == 200 assert 'ORDER BY "Hits" DESC' in data["sql"] def test_show_hits_client_side() -> None: app = server.app client = app.test_client() payload = { "table": "events", "start": "2024-01-01 00:00:00", "end": "2024-01-03 00:00:00", "graph_type": "table", "columns": ["value"], "group_by": ["user"], "aggregate": "Sum", "show_hits": False, } rv = client.post( "/api/query", data=json.dumps(payload), content_type="application/json" ) data = rv.get_json() assert rv.status_code == 200 assert "count(*) AS Hits" in data["sql"] (scubaduck) root@5b69743466e4:/workspace/scubaduck#

I'm looking at earlier code to check the test sections.

shell
n tests/test_server_timeseries.py 316:def test_order_by_samples_table() -> None: 338:def test_order_by_samples_timeseries() -> None: (scubaduck) root@5b69743466e4:/workspace/scubaduck#

I'll take a closer look at the relevant lines to review for changes.

shell
assert rv.status_code == 200 rows = sorted(data["rows"]) assert rows == [[1, 1], [2, 1]] assert all(len(r) == 2 for r in rows) def test_order_by_samples_table() -> None: app = server.app client = app.test_client() payload: dict[str, Any] = { "table": "events", "start": "2024-01-01 00:00:00", "end": "2024-01-03 00:00:00", "graph_type": "table", "order_by": "Samples", "order_dir": "DESC", "limit": 10, "columns": [], "group_by": ["user"], } rv = client.post( "/api/query", data=json.dumps(payload), content_type="application/json" ) data = rv.get_json() assert rv.status_code == 200 assert 'ORDER BY "Hits" DESC' in data["sql"] def test_order_by_samples_timeseries() -> None: app = server.app client = app.test_client() payload: dict[str, Any] = { "table": "events", "start": "2024-01-01 00:00:00", "end": "2024-01-03 00:00:00", "graph_type": "timeseries", "order_by": "Samples", "order_dir": "DESC", "x_axis": "timestamp", "granularity": "1 day", "columns": [], } rv = client.post( "/api/query", data=json.dumps(payload), content_type="application/json" ) data = rv.get_json() assert rv.status_code == 200 assert 'ORDER BY "Hits" DESC' in data["sql"] def test_show_hits_client_side() -> None: (scubaduck) root@5b69743466e4:/workspace/scubaduck#

The issue seems related to the 'group_by' check when 'Samples' is mapped to 'Hits'.

shell
(scubaduck) root@5b69743466e4:/workspace/scubaduck# head scubaduck/sample.csv timestamp,event,value,user 2024-01-01 00:00:00,login,10,alice 2024-01-01 01:00:00,logout,20,bob 2024-01-02 00:00:00,login,30,alice 2024-01-02 03:00:00,login,40,charlie (scubaduck) root@5b69743466e4:/workspace/scubaduck#

The "Unknown column: Samples" error likely results from overzealous validation—inspecting commit history could help.

The issue might occur if group_by is empty or when show_hits is inappropriately handled.

shell
(scubaduck) root@5b69743466e4:/workspace/scubaduck# ls tests | head conftest.py test_multi_table_web.py test_server.py test_server_basic.py test_server_db_types.py test_server_errors.py test_server_time.py test_server_timeseries.py test_web.py test_web_basics.py (scubaduck) root@5b69743466e4:/workspace/scubaduck#

I'll search the tests for any instances of "ORDER BY."

shell
test_server* | head tests/test_server_basic.py:166: assert "ORDER BY" not in data["sql"] tests/test_server_timeseries.py:335: assert 'ORDER BY "Hits" DESC' in data["s ql"] tests/test_server_timeseries.py:357: assert 'ORDER BY "Hits" DESC' in data["s ql"] (scubaduck) root@5b69743466e4:/workspace/scubaduck#

I'll review tests/test_server.py to check how general query features are tested.

shell
(scubaduck) root@5b69743466e4:/workspace/scubaduck# grep -n "Samples" -R tests | head tests/test_server_timeseries.py:324: "order_by": "Samples", tests/test_server_timeseries.py:346: "order_by": "Samples", (scubaduck) root@5b69743466e4:/workspace/scubaduck#

I’m checking if the tests replicate the bug—let’s run them to be sure.

shell
(scubaduck) root@5b69743466e4:/workspace/scubaduck# git status --short
(scubaduck) root@5b69743466e4:/workspace/scubaduck# pytest -q tests/test_server_ timeseries.py::test_order_by_samples_table -q
bringing up nodes... . [100%] (scubaduck) root@5b69743466e4:/workspace/scubaduck#

Hmm, maybe the bug happens with case sensitivity or mismatch in the group_by and aggregation logic.

shell
(scubaduck) root@5b69743466e4:/workspace/scubaduck# python - <<'EOF' > import json > from scubaduck import server > > app = server.app > client = app.test_client() > payload = { > "table": "events", > "graph_type": "table", > "start": "2024-01-01 00:00:00", > "end": "2024-01-03 00:00:00", > "group_by": ["user"], > "order_by": "Samples", > "order_dir": "DESC", > } > rv = client.post("/api/query", data=json.dumps(payload), content_type="applica tion/json") > print('status', rv.status_code) > print('data', rv.get_json()) > EOF status 200 data {'end': '2024-01-03 00:00:00', 'rows': [['alice', 2], ['charlie', 1], ['bob ', 1]], 'sql': 'SELECT "user", count(*) AS Hits\nFROM "events"\nWHERE "timestamp " >= \'2024-01-01 00:00:00\' AND "timestamp" <= \'2024-01-03 00:00:00\'\nGROUP B Y "user"\nORDER BY "Hits" DESC', 'start': '2024-01-01 00:00:00'} (scubaduck) root@5b69743466e4:/workspace/scubaduck#

It seems the issue arises when ordering by "Samples" without a group_by or aggregate.

shell
(scubaduck) root@5b69743466e4:/workspace/scubaduck# ls scubaduck/static/js chip_input.js table_view.js timeseries_chart.js sidebar.js time_column.js view_settings.js
(scubaduck) root@5b69743466e4:/workspace/scubaduck# grep -n "Samples" -R scubadu ck/static/js scubaduck/static/js/view_settings.js:216: samplesOpt.value = 'Samples'; scubaduck/static/js/view_settings.js:217: samplesOpt.textContent = 'Samples';
(scubaduck) root@5b69743466e4:/workspace/scubaduck# sed -n '200,240p' scubaduck/ static/js/view_settings.js graphTypeSel.addEventListener('change', updateDisplayTypeUI); document.getElementById('time_column').addEventListener('change', updateTimeFiel dVisibility); updateTimeFieldVisibility(); function loadColumns(table) { return fetch('/api/columns?table=' + encodeURIComponent(table)).then(r => r.js on()).then(cols => { const orderSelect = document.getElementById('order_by'); const xAxisSelect = document.getElementById('x_axis'); const groupsEl = document.getElementById('column_groups'); const timeColumnSelect = document.getElementById('time_column'); orderSelect.innerHTML = ''; const orderDef = document.createElement('option'); orderDef.value = ''; orderDef.textContent = '(default)'; orderSelect.appendChild(orderDef); const samplesOpt = document.createElement('option'); samplesOpt.value = 'Samples'; samplesOpt.textContent = 'Samples'; orderSelect.appendChild(samplesOpt); xAxisSelect.innerHTML = ''; const defOpt = document.createElement('option'); defOpt.value = ''; defOpt.textContent = '(default)'; xAxisSelect.appendChild(defOpt); timeColumnSelect.innerHTML = ''; const noneOpt = document.createElement('option'); noneOpt.value = ''; noneOpt.textContent = '(none)'; timeColumnSelect.appendChild(noneOpt); groupsEl.innerHTML = ''; allColumns.length = 0; stringColumns.length = 0; integerColumns.length = 0; timeColumns.length = 0; timeColumnOptions.length = 0; baseColumns.length = 0; baseStringColumns.length = 0; baseIntegerColumns.length = 0; baseTimeColumns.length = 0; baseTimeColumnOptions.length = 0; for (const k in columnTypes) delete columnTypes[k]; (scubaduck) root@5b69743466e4:/workspace/scubaduck#

I'll check the "Hits" or "Hits column" to verify the update for "Samples."

shell
(scubaduck) root@5b69743466e4:/workspace/scubaduck# grep -n "Hits" -R tests | he ad grep: tests/__pycache__/test_server_timeseries.cpython-312-pytest-8.3.5.pyc: bin ary file matches tests/test_server_errors.py:59: "columns": ["user", "Hits", "value"], tests/test_server_timeseries.py:335: assert 'ORDER BY "Hits" DESC' in data["s ql"] tests/test_server_timeseries.py:357: assert 'ORDER BY "Hits" DESC' in data["s ql"] tests/test_server_timeseries.py:378: assert "count(*) AS Hits" in data["sql"] tests/test_web_table.py:257: assert "Hits" in headers tests/test_web_table.py:346: assert headers == ["id", "Hits"] (scubaduck) root@5b69743466e4:/workspace/scubaduck#

I'll take a look at the tests in test_web_table.py that refer to Hits.

shell
assert row[2] == 25 def test_table_headers_show_aggregate(page: Any, server_url: str) -> None: run_query( page, server_url, aggregate="Avg", order_by="timestamp", ) headers = page.locator("#results th").all_inner_texts() assert "Hits" in headers assert "timestamp (avg)" in headers assert "value (avg)" in headers def test_format_number_function(page: Any, server_url: str) -> None: page.goto(server_url) page.wait_for_selector("#order_by option", state="attached") vals = page.evaluate( "() => [formatNumber(815210), formatNumber(999.999), formatNumber(0.0004 ), formatNumber(0)]" ) assert vals == ["815.21 K", "999.999", "0.000", "0"] def test_numeric_cell_nowrap(page: Any, server_url: str) -> None: run_query(page, server_url, order_by="timestamp", limit=10) whitespace = page.evaluate( "getComputedStyle(document.querySelector('#results td:nth-child(3)')).wh iteSpace" ) assert whitespace == "nowrap" def test_date_cell_nowrap(page: Any, server_url: str) -> None: run_query(page, server_url, order_by="timestamp", limit=10) whitespace = page.evaluate( "getComputedStyle(document.querySelector('#results td:nth-child(1)')).wh iteSpace" ) assert whitespace == "nowrap" def test_derived_column_query(page: Any, server_url: str) -> None: page.goto(server_url) page.wait_for_selector("#order_by option", state="attached") select_value(page, "#order_by", "timestamp") page.click("text=Columns") page.click("text=Add Derived") expr = page.query_selector("#derived_list .derived textarea") assert expr expr.fill("value * 2") page.click("text=View Settings") page.fill("#start", "2024-01-01 00:00:00") page.fill("#end", "2024-01-03 00:00:00") page.fill("#limit", "10") page.evaluate("window.lastResults = undefined") page.click("text=Dive") page.wait_for_function("window.lastResults !== undefined") data = page.evaluate("window.lastResults") assert data["rows"][0][-1] == 20 def test_derived_column_remove(page: Any, server_url: str) -> None: page.goto(server_url) page.wait_for_selector("#order_by option", state="attached") page.click("text=Columns") page.click("text=Add Derived") assert page.query_selector("#derived_list .derived button.remove") page.click("#derived_list .derived button.remove") count = page.evaluate("document.querySelectorAll('#derived_list .derived').l ength") assert count == 0 def test_sql_query_display(page: Any, server_url: str) -> None: data = run_query( page, server_url, start="2024-01-01 00:00:00", end="2024-01-02 00:00:00", order_by="timestamp", limit=10, ) sql = data["sql"] displayed = page.text_content("#sql_query") assert displayed is not None assert displayed.strip() == sql def test_table_count_no_columns(page: Any, test_dataset_server_url: str) -> None : page.goto(test_dataset_server_url) page.wait_for_selector("#order_by option", state="attached") select_value(page, "#graph_type", "table") page.click("text=Columns") page.click("#columns_all") page.click("text=View Settings") page.evaluate("groupBy.chips = ['id']; groupBy.renderChips();") select_value(page, "#aggregate", "Count") page.evaluate("window.lastResults = undefined") page.click("text=Dive") page.wait_for_function("window.lastResults !== undefined") headers = page.locator("#results th").all_inner_texts() assert headers == ["id", "Hits"] col_count = page.locator("#results th").count() row_count = page.locator("#results tr").count() assert col_count == 2 assert row_count == 3 overflow = page.evaluate( "var v=document.getElementById('view'); v.scrollWidth > v.clientWidth" ) assert not overflow (scubaduck) root@5b69743466e4:/workspace/scubaduck#

I need to keep scrolling to capture additional information.

shell
(scubaduck) root@5b69743466e4:/workspace/scubaduck# grep -n "Hits" tests/test_we b_table.py | head 257: assert "Hits" in headers 346: assert headers == ["id", "Hits"] (scubaduck) root@5b69743466e4:/workspace/scubaduck#

I'm checking around line 320 for relevant changes or anything that stands out.

shell
_web_table.py page, server_url, start="2024-01-01 00:00:00", end="2024-01-02 00:00:00", order_by="timestamp", limit=10, ) sql = data["sql"] displayed = page.text_content("#sql_query") assert displayed is not None assert displayed.strip() == sql def test_table_count_no_columns(page: Any, test_dataset_server_url: str) -> None : page.goto(test_dataset_server_url) page.wait_for_selector("#order_by option", state="attached") select_value(page, "#graph_type", "table") page.click("text=Columns") page.click("#columns_all") page.click("text=View Settings") page.evaluate("groupBy.chips = ['id']; groupBy.renderChips();") select_value(page, "#aggregate", "Count") page.evaluate("window.lastResults = undefined") page.click("text=Dive") page.wait_for_function("window.lastResults !== undefined") headers = page.locator("#results th").all_inner_texts() assert headers == ["id", "Hits"] col_count = page.locator("#results th").count() row_count = page.locator("#results tr").count() assert col_count == 2 assert row_count == 3 overflow = page.evaluate( "var v=document.getElementById('view'); v.scrollWidth > v.clientWidth" ) assert not overflow (scubaduck) root@5b69743466e4:/workspace/scubaduck#

The issue might lie in table view with group_by and order_by "Samples". Let me test further.

It seems the bug could be due to mismatched "Samples" vs. "Hits" labels on the client side.

shell
(scubaduck) root@5b69743466e4:/workspace/scubaduck# grep -n "order_by" -n scubad uck/static/js -n | head grep: scubaduck/static/js: Is a directory
(scubaduck) root@5b69743466e4:/workspace/scubaduck# grep -n "order_by" scubaduck /static/js/* scubaduck/static/js/view_settings.js:180: document.getElementById('order_by') .value = ''; scubaduck/static/js/view_settings.js:206: const orderSelect = document.getEle mentById('order_by'); scubaduck/static/js/view_settings.js:472: const orderCol = document.getElementB yId('order_by').value; scubaduck/static/js/view_settings.js:706: order_by: document.getElementById(' order_by').value, scubaduck/static/js/view_settings.js:754: if (params.order_by) sp.set('order_by ', params.order_by); scubaduck/static/js/view_settings.js:784: if (params.order_by) { scubaduck/static/js/view_settings.js:785: document.getElementById('order_by') .value = params.order_by; scubaduck/static/js/view_settings.js:849: document.getElementById('order_by').v alue = ''; scubaduck/static/js/view_settings.js:887: if (sp.has('order_by')) params.order_ by = sp.get('order_by'); (scubaduck) root@5b69743466e4:/workspace/scubaduck#

I'll review the functions around line 470 to check if they are converting correctly.

shell
(scubaduck) root@5b69743466e4:/workspace/scubaduck# sed -n '456,520p' scubaduck/ static/js/view_settings.js const agg = document.getElementById('aggregate').value.toLowerCase(); if (!(type === 'table' && agg === 'count')) { base.forEach(c => { if (!selectedColumns.includes(c)) selectedColumns.push(c); }); derivedColumns.forEach(dc => { if (dc.include && !selectedColumns.includes(dc.name)) selectedColumns.pu sh(dc.name); }); } } else { selectedColumns = base.slice(); derivedColumns.forEach(dc => { if (dc.include) selectedColumns.push(dc.name); }); } columnValues[type] = selectedColumns.slice(); const orderCol = document.getElementById('order_by').value; if (orderCol && !selectedColumns.includes(orderCol)) { selectedColumns.push(orderCol); } updateColumnsTabCount(); } function isStringColumn(name) { const t = (columnTypes[name] || '').toUpperCase(); return t.includes('CHAR') || t.includes('STRING') || t.includes('VARCHAR'); } function isIntegerColumn(name) { const t = (columnTypes[name] || '').toUpperCase(); return t.includes('INT'); } function isTimeColumn(name) { const t = (columnTypes[name] || '').toUpperCase(); if (t.includes('TIMESTAMP') || t.includes('DATE') || t.includes('TIME')) retur n true; const sel = document.getElementById('time_column').value; const xsel = document.getElementById('x_axis').value; if (name === sel || name === xsel) return true; return false; } function formatNumber(val) { if (typeof val !== 'number') val = Number(val); if (Number.isNaN(val)) return ''; if (val === 0) return '0'; const abs = Math.abs(val); if (abs > 999.999) { const units = [ {n: 1e12, s: 'T'}, {n: 1e9, s: 'B'}, {n: 1e6, s: 'M'}, {n: 1e3, s: 'K'}, ]; for (const u of units) { if (abs >= u.n) { return (val / u.n).toFixed(2) + ' ' + u.s; } } } if (abs < 0.0005) return '0.000'; if (Number.isInteger(val)) return val.toString(); return val.toFixed(3); }
static/js/view_settings.js function addFilter() { const container = document.createElement('div'); container.className = 'filter'; container.innerHTML = ` <div class="filter-row"> <select class="f-col"></select> <select class="f-op"></select> <button type="button" class="remove" onclick="this.closest('.filter').remo ve()">✖</button> </div> <div class="chip-box"> <div class="chip-input"> <input class="f-val" type="text"> <button type="button" class="chip-copy">&#x2398;</button> </div> <div class="chip-dropdown"></div> </div> `; const colSel = container.querySelector('.f-col'); colSel.innerHTML = allColumns.map(c => `<option value="${c}">${c}</option>`).j oin(''); initDropdown(colSel); function populateOps() { const opSel = container.querySelector('.f-op'); const col = colSel.value; const ops = isStringColumn(col) ? [ ['=', '='], ['!=', '!='], ['~', 'matches regex'], ['!~', 'not matches regex'], ['contains', 'contains'], ['!contains', 'not contains'], ['empty', 'empty'], ['!empty', 'not empty'], ['LIKE', 'like'], ] : [ ['=', '='], ['!=', '!='], ['<', '<'], ['>', '>'], ]; opSel.innerHTML = ops.map(o => `<option value="${o[0]}">${o[1]}</option>`).j oin(''); updateInputVis(); } function updateInputVis() { const op = container.querySelector('.f-op').value; const box = container.querySelector('.chip-box'); box.style.display = op === 'empty' || op === '!empty' ? 'none' : 'block'; } colSel.addEventListener('change', populateOps); container.querySelector('.f-op').addEventListener('change', updateInputVis); populateOps(); document.getElementById('filter_list').appendChild(container); initChipInput(container, (typed, el) => { const colEl = el.querySelector('.f-col select') || el.querySelector('.f-col' ); if (!colEl) return []; const col = colEl.value; if (!isStringColumn(col)) return []; return fetch(`/api/samples?column=${encodeURIComponent(col)}&q=${encodeURICo mponent(typed)}`) .then(r => r.json()); }); } function nextDerivedName() { let n = 1; while (true) { const name = `derived_${n}`; if (!derivedColumns.some(d => d.name === name) && !allColumns.includes(name) ) return name; n++; } } function addDerived(data = {}) { const container = document.createElement('div'); container.className = 'derived'; container.innerHTML = ` <div class="derived-row"> <select class="d-type"> <option value="aggregated">Aggregated</option> <option value="string">String</option> <option value="numeric">Numeric</option> </select> <input class="d-name" type="text"> <button type="button" class="remove" onclick="removeDerived(this)">✖</butt on> </div> <label><input type="checkbox" class="d-use" checked> Include in Query</label > <textarea class="d-expr" rows="2"></textarea> `; document.getElementById('derived_list').appendChild(container); const obj = { type: data.type || 'string', name: data.name || nextDerivedName(), expr: data.expr || '', include: data.include !== undefined ? data.include : true, el: container }; container.querySelector('.d-type').value = obj.type; container.querySelector('.d-name').value = obj.name; container.querySelector('.d-expr').value = obj.expr; container.querySelector('.d-use').checked = obj.include; ['change','input'].forEach(evt => { container.addEventListener(evt, refreshDerivedColumns); }); derivedColumns.push(obj); refreshDerivedColumns(); } function removeDerived(btn) { const el = btn.closest('.derived'); const idx = derivedColumns.findIndex(d => d.el === el); ... } orderDir = params.order_dir || 'ASC'; updateOrderDirButton(); if (params.limit !== undefined && params.limit !== null) { document.getElementById('limit').value = params.limit; limitValues[params.graph_type || 'samples'] = params.limit; limitInput.dataset.setByUser = '1'; } graphTypeSel.value = params.graph_type || 'samples'; updateDisplayTypeUI(); limitInput.value = limitValues[graphTypeSel.value]; if (params.x_axis) { document.getElementById('x_axis').value = params.x_axis; } else { document.getElementById('x_axis').value = ''; } if (params.granularity) document.getElementById('granularity').value = params. granularity; if (params.fill) document.getElementById('fill').value = params.fill; if (params.group_by) { groupBy.chips.splice(0, groupBy.chips.length, ...params.group_by); groupBy.renderChips(); } if (params.aggregate) document.getElementById('aggregate').value = params.aggr egate; document.getElementById('show_hits').checked = params.show_hits ?? true; if (params.samples_columns) columnValues.samples = params.samples_columns; if (params.table_columns) columnValues.table = params.table_columns; if (params.timeseries_columns) columnValues.timeseries = params.timeseries_col umns; document.querySelectorAll('#column_groups input').forEach(cb => { cb.checked = columnValues[graphTypeSel.value].includes(cb.value); }); updateSelectedColumns(graphTypeSel.value); const dlist = document.getElementById('derived_list'); dlist.innerHTML = ''; derivedColumns.splice(0, derivedColumns.length); if (params.derived_columns && params.derived_columns.length) { params.derived_columns.forEach(dc => addDerived(dc)); } refreshDerivedColumns(); const list = document.getElementById('filter_list'); list.innerHTML = ''; if (params.filters && params.filters.length) { params.filters.forEach(f => { addFilter(); const el = list.lastElementChild; const colSel = el.querySelector('.f-col select') || el.querySelector('.f-c ol'); colSel.value = f.column; colSel.dispatchEvent(new Event('change')); el.querySelector('.f-op').value = f.op; el.querySelector('.f-op').dispatchEvent(new Event('change')); if (f.value !== null && f.op !== 'empty' && f.op !== '!empty') { const values = Array.isArray(f.value) ? f.value : [f.value]; values.forEach(v => el.addChip(v)); el.renderChips(); } }); } else { addFilter(); } } function resetViewSettings() { orderDir = 'ASC'; updateOrderDirButton(); document.getElementById('order_by').value = ''; document.getElementById('start').value = ''; document.getElementById('end').value = ''; document.getElementById('time_unit').value = 's'; document.getElementById('granularity').value = 'Auto'; document.getElementById('fill').value = '0'; document.getElementById('aggregate').value = 'Count'; document.getElementById('show_hits').checked = true; document.getElementById('x_axis').value = ''; groupBy.chips.splice(0, groupBy.chips.length); groupBy.renderChips(); const dlist = document.getElementById('derived_list'); dlist.innerHTML = ''; derivedColumns.splice(0, derivedColumns.length); refreshDerivedColumns(); const flist = document.getElementById('filter_list'); flist.innerHTML = ''; addFilter(); document.getElementById('graph_type').value = 'samples'; limitValues.samples = defaultLimit; limitValues.table = defaultLimit; limitValues.timeseries = 7; limitInput.dataset.setByUser = ''; updateDisplayTypeUI(); document.querySelectorAll('#column_groups input').forEach(cb => { cb.checked = true; }); updateSelectedColumns(); } function parseSearch() { const sp = new URLSearchParams(window.location.search); const params = {}; if (sp.has('table')) params.table = sp.get('table'); if (sp.has('time_column')) params.time_column = sp.get('time_column'); if (sp.has('time_unit')) params.time_unit = sp.get('time_unit'); if (sp.has('start')) params.start = sp.get('start'); if (sp.has('end')) params.end = sp.get('end'); if (sp.has('order_by')) params.order_by = sp.get('order_by'); if (sp.has('order_dir')) params.order_dir = sp.get('order_dir'); if (sp.has('limit')) params.limit = parseInt(sp.get('limit'), 10); if (sp.has('samples_columns')) params.samples_columns = sp.get('samples_column s').split(',').filter(c => c); if (sp.has('table_columns')) params.table_columns = sp.get('table_columns').sp lit(',').filter(c => c); if (sp.has('timeseries_columns')) params.timeseries_columns = sp.get('timeseri es_columns').split(',').filter(c => c); if (sp.has('filters')) { try { params.filters = JSON.parse(sp.get('filters')); } catch(e) { params.fi lters = []; } } if (sp.has('graph_type')) params.graph_type = sp.get('graph_type'); if (sp.has('group_by')) params.group_by = sp.get('group_by').split(',').filter (c => c); if (sp.has('aggregate')) params.aggregate = sp.get('aggregate'); if (sp.has('show_hits')) params.show_hits = sp.get('show_hits') === '1'; if (sp.has('x_axis')) params.x_axis = sp.get('x_axis'); (scubaduck) root@5b69743466e4:/workspace/scubaduck#
(scubaduck) root@5b69743466e4:/workspace/scubaduck# sed -n '900,1060p' scubaduck /static/js/view_settings.js if (sp.has('x_axis')) params.x_axis = sp.get('x_axis'); if (sp.has('granularity')) params.granularity = sp.get('granularity'); if (sp.has('fill')) params.fill = sp.get('fill'); if (sp.has('derived_columns')) { try { params.derived_columns = JSON.parse(sp.get('derived_columns')); } catc h(e) { params.derived_columns = []; } } return params; } function initFromUrl() { const params = parseSearch(); history.replaceState(params, '', paramsToSearch(params)); applyParams(params); dive(false); } window.addEventListener('popstate', e => { const params = e.state || parseSearch(); applyParams(params); dive(false); }); function setSelectValue(selector, value) { const el = typeof selector === 'string' ? document.querySelector(selector) : s elector; if (el) { const select = el.tagName === 'SELECT' ? el : el.querySelector('select'); if (select) { select.value = value; select.dispatchEvent(new Event('change')); } } }
(scubaduck) root@5b69743466e4:/workspace/scubaduck# grep -n "api/query" -n scuba duck/static/js/view_settings.js 683: fetch('/api/query', {method:'POST', headers:{'Content-Type':'application/j son'}, body:JSON.stringify(payload)})
(scubaduck) root@5b69743466e4:/workspace/scubaduck# sed -n '640,720p' scubaduck/ static/js/view_settings.js } function refreshDerivedColumns() { allColumns.splice(0, allColumns.length, ...baseColumns); stringColumns.splice(0, stringColumns.length, ...baseStringColumns); integerColumns.splice(0, integerColumns.length, ...baseIntegerColumns); timeColumns.splice(0, timeColumns.length, ...baseTimeColumns); timeColumnOptions.splice(0, timeColumnOptions.length, ...baseTimeColumnOptions ); derivedColumns.forEach(d => { d.type = d.el.querySelector('.d-type').value; d.name = d.el.querySelector('.d-name').value; d.expr = d.el.querySelector('.d-expr').value; d.include = d.el.querySelector('.d-use').checked; allColumns.push(d.name); columnTypes[d.name] = d.type; if (d.type === 'string') { stringColumns.push(d.name); } else { integerColumns.push(d.name); timeColumnOptions.push(d.name); } }); updateSelectedColumns(); } let lastQueryTime = 0; let queryStart = 0; function dive(push=true) { const params = collectParams(); if (push) { history.pushState(params, '', paramsToSearch(params)); } const payload = Object.assign({}, params); const dcMap = {}; (params.derived_columns || []).forEach(d => { if (d.include) dcMap[d.name] = d.expr; }); payload.derived_columns = dcMap; const view = document.getElementById('view'); view.innerHTML = '<p>Loading...</p>'; window.lastResults = undefined; queryStart = performance.now(); fetch('/api/query', {method:'POST', headers:{'Content-Type':'application/json' }, body:JSON.stringify(payload)}) .then(async r => { const data = await r.json(); if (!r.ok) throw data; return data; }) .then(data => { lastQueryTime = Math.round(performance.now() - queryStart); showResults(data); }) .catch(err => { showError(err); }); } function collectParams() { updateSelectedColumns(); const payload = { table: document.getElementById('table').value, time_column: document.getElementById('time_column').value, time_unit: document.getElementById('time_unit').value, start: document.getElementById('start').value, end: document.getElementById('end').value, order_by: document.getElementById('order_by').value, order_dir: orderDir, limit: parseInt(document.getElementById('limit').value, 10), columns: selectedColumns.filter(c => c !== 'Hits' && !derivedColumns.some(dc => dc.name === c) ), samples_columns: columnValues.samples.slice(), table_columns: columnValues.table.slice(), timeseries_columns: columnValues.timeseries.slice(), graph_type: graphTypeSel.value, filters: Array.from(document.querySelectorAll('#filters .filter')).map(f => { const chips = f.chips || []; const op = f.querySelector('.f-op').value; let value = null; if (op !== 'empty' && op !== '!empty') { (scubaduck) root@5b69743466e4:/workspace/scubaduck#

I'm looking through more lines to get the full context.

shell
(scubaduck) root@5b69743466e4:/workspace/scubaduck# sed -n '720,840p' scubaduck/ static/js/view_settings.js if (op !== 'empty' && op !== '!empty') { value = chips.length === 0 ? null : (chips.length === 1 ? chips[0] : chi ps); } const colSel = f.querySelector('.f-col select') || f.querySelector('.f-col '); return {column: colSel.value, op, value}; }), derived_columns: Array.from(document.querySelectorAll('#derived_list .derive d')).map(d => ({ type: d.querySelector('.d-type').value, name: d.querySelector('.d-name').value, expr: d.querySelector('.d-expr').value, include: d.querySelector('.d-use').checked, })) }; if (graphTypeSel.value === 'table' || graphTypeSel.value === 'timeseries') { payload.group_by = groupBy.chips || []; payload.aggregate = document.getElementById('aggregate').value; payload.show_hits = document.getElementById('show_hits').checked; } if (graphTypeSel.value === 'timeseries') { const xval = document.getElementById('x_axis').value; if (xval) payload.x_axis = xval; payload.granularity = document.getElementById('granularity').value; payload.fill = document.getElementById('fill').value; } return payload; } function paramsToSearch(params) { const sp = new URLSearchParams(); if (params.table) sp.set('table', params.table); if (params.time_column) sp.set('time_column', params.time_column); if (params.time_unit) sp.set('time_unit', params.time_unit); if (params.start) sp.set('start', params.start); if (params.end) sp.set('end', params.end); if (params.order_by) sp.set('order_by', params.order_by); if (params.order_dir) sp.set('order_dir', params.order_dir); if (params.limit !== null && params.limit !== undefined) sp.set('limit', param s.limit); if (params.samples_columns && params.samples_columns.length) sp.set('samples_c olumns', params.samples_columns.join(',')); if (params.table_columns && params.table_columns.length) sp.set('table_columns ', params.table_columns.join(',')); if (params.timeseries_columns && params.timeseries_columns.length) sp.set('tim eseries_columns', params.timeseries_columns.join(',')); if (params.filters && params.filters.length) sp.set('filters', JSON.stringify( params.filters)); if (params.derived_columns && params.derived_columns.length) sp.set('derived_c olumns', JSON.stringify(params.derived_columns)); if (params.graph_type) sp.set('graph_type', params.graph_type); if (params.graph_type === 'table' || params.graph_type === 'timeseries') { if (params.group_by && params.group_by.length) sp.set('group_by', params.gro up_by.join(',')); if (params.aggregate) sp.set('aggregate', params.aggregate); if (params.show_hits) sp.set('show_hits', '1'); } if (params.graph_type === 'timeseries') { if (params.x_axis) sp.set('x_axis', params.x_axis); if (params.granularity) sp.set('granularity', params.granularity); if (params.fill) sp.set('fill', params.fill); } const qs = sp.toString(); return qs ? '?' + qs : ''; } function applyParams(params) { if (params.table) document.getElementById('table').value = params.table; document.getElementById('time_column').value = params.time_column || defaultTi meColumn; updateTimeFieldVisibility(); if (params.time_unit) document.getElementById('time_unit').value = params.time _unit; document.getElementById('start').value = params.start || ''; document.getElementById('end').value = params.end || ''; if (params.order_by) { document.getElementById('order_by').value = params.order_by; } orderDir = params.order_dir || 'ASC'; updateOrderDirButton(); if (params.limit !== undefined && params.limit !== null) { document.getElementById('limit').value = params.limit; limitValues[params.graph_type || 'samples'] = params.limit; limitInput.dataset.setByUser = '1'; } graphTypeSel.value = params.graph_type || 'samples'; updateDisplayTypeUI(); limitInput.value = limitValues[graphTypeSel.value]; if (params.x_axis) { document.getElementById('x_axis').value = params.x_axis; } else { document.getElementById('x_axis').value = ''; } if (params.granularity) document.getElementById('granularity').value = params. granularity; if (params.fill) document.getElementById('fill').value = params.fill; if (params.group_by) { groupBy.chips.splice(0, groupBy.chips.length, ...params.group_by); groupBy.renderChips(); } if (params.aggregate) document.getElementById('aggregate').value = params.aggr egate; document.getElementById('show_hits').checked = params.show_hits ?? true; if (params.samples_columns) columnValues.samples = params.samples_columns; if (params.table_columns) columnValues.table = params.table_columns; if (params.timeseries_columns) columnValues.timeseries = params.timeseries_col umns; document.querySelectorAll('#column_groups input').forEach(cb => { cb.checked = columnValues[graphTypeSel.value].includes(cb.value); }); updateSelectedColumns(graphTypeSel.value); const dlist = document.getElementById('derived_list'); dlist.innerHTML = ''; derivedColumns.splice(0, derivedColumns.length); if (params.derived_columns && params.derived_columns.length) { params.derived_columns.forEach(dc => addDerived(dc)); } refreshDerivedColumns(); const list = document.getElementById('filter_list'); list.innerHTML = ''; if (params.filters && params.filters.length) { params.filters.forEach(f => { addFilter(); const el = list.lastElementChild; const colSel = el.querySelector('.f-col select') || el.querySelector('.f-c ol'); colSel.value = f.column; colSel.dispatchEvent(new Event('change')); el.querySelector('.f-op').value = f.op; el.querySelector('.f-op').dispatchEvent(new Event('change')); if (f.value !== null && f.op !== 'empty' && f.op !== '!empty') { const values = Array.isArray(f.value) ? f.value : [f.value]; values.forEach(v => el.addChip(v)); el.renderChips(); } }); (scubaduck) root@5b69743466e4:/workspace/scubaduck#

I'll continue scrolling to capture more details.

shell
(scubaduck) root@5b69743466e4:/workspace/scubaduck# sed -n '840,960p' scubaduck/ static/js/view_settings.js }); } else { addFilter(); } } function resetViewSettings() { orderDir = 'ASC'; updateOrderDirButton(); document.getElementById('order_by').value = ''; document.getElementById('start').value = ''; document.getElementById('end').value = ''; document.getElementById('time_unit').value = 's'; document.getElementById('granularity').value = 'Auto'; document.getElementById('fill').value = '0'; document.getElementById('aggregate').value = 'Count'; document.getElementById('show_hits').checked = true; document.getElementById('x_axis').value = ''; groupBy.chips.splice(0, groupBy.chips.length); groupBy.renderChips(); const dlist = document.getElementById('derived_list'); dlist.innerHTML = ''; derivedColumns.splice(0, derivedColumns.length); refreshDerivedColumns(); const flist = document.getElementById('filter_list'); flist.innerHTML = ''; addFilter(); document.getElementById('graph_type').value = 'samples'; limitValues.samples = defaultLimit; limitValues.table = defaultLimit; limitValues.timeseries = 7; limitInput.dataset.setByUser = ''; updateDisplayTypeUI(); document.querySelectorAll('#column_groups input').forEach(cb => { cb.checked = true; }); updateSelectedColumns(); } function parseSearch() { const sp = new URLSearchParams(window.location.search); const params = {}; if (sp.has('table')) params.table = sp.get('table'); if (sp.has('time_column')) params.time_column = sp.get('time_column'); if (sp.has('time_unit')) params.time_unit = sp.get('time_unit'); if (sp.has('start')) params.start = sp.get('start'); if (sp.has('end')) params.end = sp.get('end'); if (sp.has('order_by')) params.order_by = sp.get('order_by'); if (sp.has('order_dir')) params.order_dir = sp.get('order_dir'); if (sp.has('limit')) params.limit = parseInt(sp.get('limit'), 10); if (sp.has('samples_columns')) params.samples_columns = sp.get('samples_column s').split(',').filter(c => c); if (sp.has('table_columns')) params.table_columns = sp.get('table_columns').sp lit(',').filter(c => c); if (sp.has('timeseries_columns')) params.timeseries_columns = sp.get('timeseri es_columns').split(',').filter(c => c); if (sp.has('filters')) { try { params.filters = JSON.parse(sp.get('filters')); } catch(e) { params.fi lters = []; } } if (sp.has('graph_type')) params.graph_type = sp.get('graph_type'); if (sp.has('group_by')) params.group_by = sp.get('group_by').split(',').filter (c => c); if (sp.has('aggregate')) params.aggregate = sp.get('aggregate'); if (sp.has('show_hits')) params.show_hits = sp.get('show_hits') === '1'; if (sp.has('x_axis')) params.x_axis = sp.get('x_axis'); if (sp.has('granularity')) params.granularity = sp.get('granularity'); if (sp.has('fill')) params.fill = sp.get('fill'); if (sp.has('derived_columns')) { try { params.derived_columns = JSON.parse(sp.get('derived_columns')); } catc h(e) { params.derived_columns = []; } } return params; } function initFromUrl() { const params = parseSearch(); history.replaceState(params, '', paramsToSearch(params)); applyParams(params); dive(false); } window.addEventListener('popstate', e => { const params = e.state || parseSearch(); applyParams(params); dive(false); }); function setSelectValue(selector, value) { const el = typeof selector === 'string' ? document.querySelector(selector) : s elector; if (el) { const select = el.tagName === 'SELECT' ? el : el.querySelector('select'); if (select) { select.value = value; select.dispatchEvent(new Event('change')); } } } (scubaduck) root@5b69743466e4:/workspace/scubaduck#

I’m scrolling back to locate the results that need to be sent.

shell
(scubaduck) root@5b69743466e4:/workspace/scubaduck# grep -n "Hits" -n scubaduck/ static/js/view_settings.js 455: if (document.getElementById('show_hits').checked) selectedColumns.push(' Hits'); 710: c !== 'Hits' && !derivedColumns.some(dc => dc.name === c) (scubaduck) root@5b69743466e4:/workspace/scubaduck#

I'll focus on lines 455 to 470 to check for relevant changes.

shell
(scubaduck) root@5b69743466e4:/workspace/scubaduck# sed -n '448,475p' scubaduck/ static/js/view_settings.js const cb = document.querySelector(`#column_groups input[value="${name}"]`); if (!cb || !cb.checked) return false; if (type === 'table' && isStringColumn(name)) return false; return true; }); if (type === 'table' || type === 'timeseries') { selectedColumns = groupBy.chips.slice(); if (document.getElementById('show_hits').checked) selectedColumns.push('Hits '); const agg = document.getElementById('aggregate').value.toLowerCase(); if (!(type === 'table' && agg === 'count')) { base.forEach(c => { if (!selectedColumns.includes(c)) selectedColumns.push(c); }); derivedColumns.forEach(dc => { if (dc.include && !selectedColumns.includes(dc.name)) selectedColumns.pu sh(dc.name); }); } } else { selectedColumns = base.slice(); derivedColumns.forEach(dc => { if (dc.include) selectedColumns.push(dc.name); }); } columnValues[type] = selectedColumns.slice(); const orderCol = document.getElementById('order_by').value; if (orderCol && !selectedColumns.includes(orderCol)) { selectedColumns.push(orderCol); } (scubaduck) root@5b69743466e4:/workspace/scubaduck#

Looks like the issue arises if show_hits isn't checked—Hits might not be included.

The bug likely occurs before the query: adding Group By then ordering by Samples causes error.

shell
(scubaduck) root@5b69743466e4:/workspace/scubaduck# grep -n "group_by" -n tests/ test_server* | head tests/test_server_db_types.py:183:def test_sqlite_boolean_group_by(tmp_path: Pat h) -> None: tests/test_server_db_types.py:204: "group_by": ["id"], tests/test_server_errors.py:38: "group_by": ["user"], tests/test_server_errors.py:60: "group_by": ["user"], tests/test_server_errors.py:71:def test_samples_view_rejects_group_by() -> None: tests/test_server_errors.py:77: "group_by": ["user"], tests/test_server_timeseries.py:9:def test_group_by_table() -> None: tests/test_server_timeseries.py:20: "group_by": ["user"], tests/test_server_timeseries.py:44: "group_by": ["user"], tests/test_server_timeseries.py:70: "group_by": ["user"], (scubaduck) root@5b69743466e4:/workspace/scubaduck#

I'm checking the 'test_server_errors.py' file around the 'order_by' references.

shell
"filters": [], } rv = client.post( "/api/query", data=json.dumps(payload), content_type="application/json" ) data = rv.get_json() assert rv.status_code == 400 assert "error" in data def test_query_error_returns_sql_and_traceback() -> None: app = server.app client = app.test_client() payload = { "table": "events", "start": "2024-01-01 00:00:00", "end": "2024-01-03 00:00:00", "columns": ["event"], "group_by": ["user"], "aggregate": "avg", } rv = client.post( "/api/query", data=json.dumps(payload), content_type="application/json" ) data = rv.get_json() assert rv.status_code == 400 assert "error" in data def test_table_unknown_column_error() -> None: app = server.app client = app.test_client() payload = { "table": "events", "start": "2024-01-01 00:00:00", "end": "2024-01-03 00:00:00", "graph_type": "table", "order_by": "timestamp", "limit": 100, "columns": ["user", "Hits", "value"], "group_by": ["user"], "aggregate": "Count", "show_hits": True, } rv = client.post( "/api/query", data=json.dumps(payload), content_type="application/json" ) _data = rv.get_json() assert rv.status_code == 200 def test_samples_view_rejects_group_by() -> None: app = server.app client = app.test_client() payload = { "table": "events", "graph_type": "samples", "group_by": ["user"], "columns": ["timestamp"], } rv = client.post( (scubaduck) root@5b69743466e4:/workspace/scubaduck#

I'll scroll back to section 38 and double-check for relevant details.

shell
erver_errors.py "end": "2024-01-03 00:00:00", "columns": ["event"], "group_by": ["user"], "aggregate": "avg", } rv = client.post( "/api/query", data=json.dumps(payload), content_type="application/json" ) data = rv.get_json() assert rv.status_code == 400 assert "error" in data def test_table_unknown_column_error() -> None: app = server.app client = app.test_client() payload = { "table": "events", "start": "2024-01-01 00:00:00", "end": "2024-01-03 00:00:00", "graph_type": "table", "order_by": "timestamp", "limit": 100, "columns": ["user", "Hits", "value"], "group_by": ["user"], "aggregate": "Count", "show_hits": True, } rv = client.post( "/api/query", data=json.dumps(payload), content_type="application/json" ) _data = rv.get_json() assert rv.status_code == 200 (scubaduck) root@5b69743466e4:/workspace/scubaduck#

The bug might not replicate due to recent fixes. Opening commit history to confirm.

shell
commit e08b86baa34cfb3a5182ad3681596db58c2b1642 (HEAD -> work, main) Author: Edward Z. Yang <ezyang@mit.edu> Date: Thu May 22 23:01:28 2025 -0400 fix(dropdown): ensure selected item is scrolled into view (#154) scubaduck/static/js/view_settings.js | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) (scubaduck) root@5b69743466e4:/workspace/scubaduck#

I’ll check the previous commit to validate the one from 2025.

shell
(scubaduck) root@5b69743466e4:/workspace/scubaduck# git log -5 --oneline e08b86b (HEAD -> work, main) fix(dropdown): ensure selected item is scrolled int o view (#154) 2451f86 fix(server): enforce series limit in sparse timeseries (#153) b2bd0b9 feat(server): suggest time unit in errors (#152) 6afebd6 fix(frontend): prevent clipped rotated time labels (#150) 079223c feat(ui): scroll dropdown to selected option (#151) (scubaduck) root@5b69743466e4:/workspace/scubaduck#

I'll look for "Samples" in the commit messages to track the bug fix.

shell
commit 2451f864f3fec8ad4422f119f55359deeef24f66 Author: Edward Z. Yang <ezyang@mit.edu> Date: Thu May 22 23:01:18 2025 -0400 fix(server): enforce series limit in sparse timeseries (#153) diff --git a/scubaduck/server.py b/scubaduck/server.py index a3c7268..d292c4a 100644 --- a/scubaduck/server.py +++ b/scubaduck/server.py (scubaduck) root@5b69743466e4:/workspace/scubaduck#

I'll search for any commits mentioning "Samples" to track changes related to them.

shell
(scubaduck) root@5b69743466e4:/workspace/scubaduck# git log --grep "Samples" -n 5 commit 9b4de028df02ef944bae7bff88c2845de91abf0d Author: Edward Z. Yang <ezyang@mit.edu> Date: Thu May 22 22:16:07 2025 -0400 feat(server): add Samples order by and always count hits (#143) (scubaduck) root@5b69743466e4:/workspace/scubaduck#

I'll review commit 9b4de028 to understand what changes were made.

shell
(scubaduck) root@5b69743466e4:/workspace/scubaduck# git show --stat 9b4de028df02 ef944bae7bff88c2845de91abf0d commit 9b4de028df02ef944bae7bff88c2845de91abf0d Author: Edward Z. Yang <ezyang@mit.edu> Date: Thu May 22 22:16:07 2025 -0400 feat(server): add Samples order by and always count hits (#143) scubaduck/server.py | 13 ++++-- scubaduck/static/js/table_view.js | 9 ++++ scubaduck/static/js/view_settings.js | 8 ++++ tests/test_server_db_types.py | 2 +- tests/test_server_errors.py | 3 +- tests/test_server_timeseries.py | 80 ++++++++++++++++++++++++++++++++---- 6 files changed, 101 insertions(+), 14 deletions(-) (scubaduck) root@5b69743466e4:/workspace/scubaduck#

I'll take a look at the differences in this commit.

shell
(scubaduck) root@5b69743466e4:/workspace/scubaduck# git show 9b4de028df02ef944ba e7bff88c2845de91abf0d | head -n 40 commit 9b4de028df02ef944bae7bff88c2845de91abf0d Author: Edward Z. Yang <ezyang@mit.edu> Date: Thu May 22 22:16:07 2025 -0400 feat(server): add Samples order by and always count hits (#143) diff --git a/scubaduck/server.py b/scubaduck/server.py index dd9ff6c..1a82eaa 100644 --- a/scubaduck/server.py +++ b/scubaduck/server.py @@ -304,14 +304,16 @@ def build_query(params: QueryParams, column_types: Dict[st r, str] | None = None) continue select_parts.append(f"{agg_expr(col)} AS {_quote(col)}") selected_for_order.add(col) - if params.show_hits: - select_parts.insert(len(group_cols), "count(*) AS Hits") - selected_for_order.add("Hits") + select_parts.insert(len(group_cols), "count(*) AS Hits") + selected_for_order.add("Hits") else: select_parts.extend(_quote(c) for c in params.columns) selected_for_order.update(params.columns) - order_by = params.order_by if params.order_by in selected_for_order else No ne + order_by = params.order_by + if order_by == "Samples": + order_by = "Hits" + order_by = order_by if order_by in selected_for_order else None if has_agg and params.derived_columns: inner_params = replace( @@ -513,6 +515,8 @@ def create_app(db_file: str | Path | None = None) -> Flask: time_column=payload.get("time_column", "timestamp"), time_unit=payload.get("time_unit", "s"), ) + if params.order_by == "Samples": + params.order_by = "Hits" for f in payload.get("filters", []): params.filters.append(Filter(f["column"], f["op"], f.get("value"))) (scubaduck) root@5b69743466e4:/workspace/scubaduck#

I’m continuing to scroll through the diff to examine further changes.

shell
@@ -541,6 +545,7 @@ def create_app(db_file: str | Path | None = None) -> Flask: valid_cols = set(column_types.keys()) valid_cols.update(params.derived_columns.keys()) + valid_cols.add("Hits") if params.graph_type == "timeseries": if params.x_axis is None: params.x_axis = params.time_column diff --git a/scubaduck/static/js/table_view.js b/scubaduck/static/js/table_view. js index f417e7c..ff11719 100644 --- a/scubaduck/static/js/table_view.js +++ b/scubaduck/static/js/table_view.js @@ -123,6 +123,15 @@ function handleSort(e) { function showResults(data) { window.lastResults = data; + const hideHits = + (graphTypeSel.value === "table" || graphTypeSel.value === "timeseries") && + !document.getElementById("show_hits").checked; + if (hideHits && data.rows.length) { + const groupCount = + (graphTypeSel.value === "timeseries" ? 1 : 0) + + ((groupBy.chips || []).length || 0); + data.rows.forEach((r) => r.splice(groupCount, 1)); + } const view = document.getElementById("view"); if (graphTypeSel.value === "timeseries") { showTimeSeries(data); diff --git a/scubaduck/static/js/view_settings.js b/scubaduck/static/js/view_set tings.js index 13d790a..329a4ae 100644 --- a/scubaduck/static/js/view_settings.js +++ b/scubaduck/static/js/view_settings.js @@ -200,6 +200,14 @@ function loadColumns(table) { const groupsEl = document.getElementById('column_groups'); const timeColumnSelect = document.getElementById('time_column'); orderSelect.innerHTML = ''; + const orderDef = document.createElement('option'); + orderDef.value = ''; + orderDef.textContent = '(default)'; + orderSelect.appendChild(orderDef); + const samplesOpt = document.createElement('option'); + samplesOpt.value = 'Samples'; + samplesOpt.textContent = 'Samples'; + orderSelect.appendChild(samplesOpt); xAxisSelect.innerHTML = ''; const defOpt = document.createElement('option'); defOpt.value = ''; diff --git a/tests/test_server_db_types.py b/tests/test_server_db_types.py index baaf19b..e4841a1 100644 --- a/tests/test_server_db_types.py +++ b/tests/test_server_db_types.py @@ -182,7 +182,7 @@ def test_sqlite_boolean_group_by(tmp_path: Path) -> None: data = rv.get_json() assert rv.status_code == 200 rows = sorted(data["rows"]) # order can vary - assert rows == [[1, 0.5], [2, 1.0]] + assert rows == [[1, 2, 0.5], [2, 1, 1.0]] def test_envvar_db(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: diff --git a/tests/test_server_errors.py b/tests/test_server_errors.py index d0a1da5..a19c554 100644 --- a/tests/test_server_errors.py +++ b/tests/test_server_errors.py @@ -63,8 +63,7 @@ def test_table_unknown_column_error() -> None: "/api/query", data=json.dumps(payload), content_type="application/json" ) data = rv.get_json() - assert rv.status_code == 400 - assert "Unknown column" in data["error"] + assert rv.status_code == 200 def test_samples_view_rejects_group_by() -> None: diff --git a/tests/test_server_timeseries.py b/tests/test_server_timeseries.py index f8d5f8d..cb44f0f 100644 --- a/tests/test_server_timeseries.py +++ b/tests/test_server_timeseries.py @@ -9,7 +9,7 @@ from scubaduck import server def test_group_by_table() -> None: app = server.app client = app.test_client() - payload = { + payload: dict[str, Any] = { "table": "events", "start": "2024-01-01 00:00:00", "end": "2024-01-03 00:00:00", @@ -33,7 +33,7 @@ def test_group_by_table() -> None: def test_table_avg_with_timestamp() -> None: app = server.app client = app.test_client() - payload = { + payload: dict[str, Any] = { "table": "events", "start": "2024-01-01 00:00:00", "end": "2024-01-03 00:00:00", @@ -54,14 +54,14 @@ def test_table_avg_with_timestamp() -> None: assert rows[0][0] == "alice" from dateutil import parser - ts = parser.parse(rows[0][1]).replace(tzinfo=None) + ts = parser.parse(rows[0][2]).replace(tzinfo=None) assert ts == parser.parse("2024-01-01 12:00:00") def test_timeseries_basic() -> None: app = server.app client = app.test_client() - payload = { + payload: dict[str, Any] = { "table": "events", "start": "2024-01-01 00:00:00", "end": "2024-01-03 00:00:00", @@ -84,7 +84,7 @@ def test_timeseries_basic() -> None: def test_timeseries_orders_by_xaxis() -> None: app = server.app client = app.test_client() - payload = { + payload: dict[str, Any] = { "table": "events", (scubaduck) root@5b69743466e4:/workspace/scubaduck#

I need to scroll a bit more to see everything I need.

shell
@@ -126,12 +126,13 @@ def test_timeseries_count_no_columns() -> None: assert len(rows) == 2 assert rows[0][1] == 2 assert rows[1][1] == 2 + assert len(rows[0]) == 3 def test_timeseries_limit_applies_to_series() -> None: app = server.app client = app.test_client() - payload = { + payload: dict[str, Any] = { "table": "events", "start": "2024-01-01 00:00:00", "end": "2024-01-03 00:00:00", @@ -246,7 +247,7 @@ def test_timeseries_derived_column() -> None: data = rv.get_json() assert rv.status_code == 200 rows = data["rows"] - assert all(r[2] == r[1] * 2 for r in rows) + assert all(r[3] == r[2] * 2 for r in rows) def test_reserved_word_column() -> None: @@ -265,3 +266,68 @@ def test_reserved_word_column() -> None: assert rv.status_code == 200 assert len(data["rows"]) == 2 assert data["rows"][0][1] == "x" + + +def test_order_by_samples_table() -> None: + app = server.app + client = app.test_client() + payload: dict[str, Any] = { + "table": "events", + "start": "2024-01-01 00:00:00", + "end": "2024-01-03 00:00:00", + "graph_type": "table", + "order_by": "Samples", + "order_dir": "DESC", + "limit": 10, + "columns": [], + "group_by": ["user"], + } + rv = client.post( + "/api/query", data=json.dumps(payload), content_type="application/json" + ) + data = rv.get_json() + assert rv.status_code == 200 + assert 'ORDER BY "Hits" DESC' in data["sql"] + + +def test_order_by_samples_timeseries() -> None: + app = server.app + client = app.test_client() + payload: dict[str, Any] = { + "table": "events", + "start": "2024-01-01 00:00:00", + "end": "2024-01-03 00:00:00", + "graph_type": "timeseries", + "order_by": "Samples", + "order_dir": "DESC", + "x_axis": "timestamp", + "granularity": "1 day", + "columns": [], + } + rv = client.post( + "/api/query", data=json.dumps(payload), content_type="application/json" + ) + data = rv.get_json() + assert rv.status_code == 200 + assert 'ORDER BY "Hits" DESC' in data["sql"] + + +def test_show_hits_client_side() -> None: + app = server.app + client = app.test_client() + payload = { + "table": "events", + "start": "2024-01-01 00:00:00", + "end": "2024-01-03 00:00:00", + "graph_type": "table", + "columns": ["value"], + "group_by": ["user"], + "aggregate": "Sum", + "show_hits": False, + } + rv = client.post( + "/api/query", data=json.dumps(payload), content_type="application/json" + ) + data = rv.get_json() + assert rv.status_code == 200 + assert "count(*) AS Hits" in data["sql"] (scubaduck) root@5b69743466e4:/workspace/scubaduck#

I'll check commits after 9b4de028 for changes related to "Hits" or "Samples" that might caused the bug.

shell
e08b86b fix(dropdown): ensure selected item is scrolled into view (#154) 2451f86 fix(server): enforce series limit in sparse timeseries (#153) b2bd0b9 feat(server): suggest time unit in errors (#152) 6afebd6 fix(frontend): prevent clipped rotated time labels (#150) 079223c feat(ui): scroll dropdown to selected option (#151) 80a8ced fix(ui): show count series for numeric timestamps (#148) e90ea2e fix(server): json encode bytes (#149) a8074f6 fix(server): drop duplicate count column in table view (#147) 2ac2fc5 fix(ui): reset view settings on table change (#144) 9b4de02 feat(server): add Samples order by and always count hits (#143) (scubaduck) root@5b69743466e4:/workspace/scubaduck#

Looks like commit a8074f6 might have removed the hits column or something similar.

shell
(scubaduck) root@5b69743466e4:/workspace/scubaduck# git show --stat a8074f6 commit a8074f6ccca3f3c7a48bdc2aa762ea90177b1b87 Author: Edward Z. Yang <ezyang@mit.edu> Date: Thu May 22 22:28:01 2025 -0400 fix(server): drop duplicate count column in table view (#147) scubaduck/server.py | 5 +++-- tests/test_server_errors.py | 2 +- tests/test_server_timeseries.py | 21 +++++++++++++++++++++ 3 files changed, 25 insertions(+), 3 deletions(-) (scubaduck) root@5b69743466e4:/workspace/scubaduck#

I'll check the patch to understand the changes made.

shell
commit a8074f6ccca3f3c7a48bdc2aa762ea90177b1b87 Author: Edward Z. Yang <ezyang@mit.edu> Date: Thu May 22 22:28:01 2025 -0400 fix(server): drop duplicate count column in table view (#147) diff --git a/scubaduck/server.py b/scubaduck/server.py index 1a82eaa..5f48427 100644 --- a/scubaduck/server.py +++ b/scubaduck/server.py @@ -296,8 +296,9 @@ def build_query(params: QueryParams, column_types: Dict[str, str] | None = None) return f"{agg}({expr})" if agg == "count": - select_parts.append("count(*) AS Count") - selected_for_order.add("Count") + if params.graph_type != "table": + select_parts.append("count(*) AS Count") + selected_for_order.add("Count") else: for col in params.columns: if col in group_cols: diff --git a/tests/test_server_errors.py b/tests/test_server_errors.py index a19c554..1da922b 100644 --- a/tests/test_server_errors.py +++ b/tests/test_server_errors.py @@ -62,7 +62,7 @@ def test_table_unknown_column_error() -> None: rv = client.post( "/api/query", data=json.dumps(payload), content_type="application/json" ) - data = rv.get_json() + _ = rv.get_json() assert rv.status_code == 200 diff --git a/tests/test_server_timeseries.py b/tests/test_server_timeseries.py index cb44f0f..205fe7b 100644 --- a/tests/test_server_timeseries.py +++ b/tests/test_server_timeseries.py @@ -268,6 +268,27 @@ def test_reserved_word_column() -> None: assert data["rows"][0][1] == "x" +def test_table_count_group_by_extra() -> None: + app = server.create_app("TEST") + client = app.test_client() + payload = { + "table": "extra", + "graph_type": "table", + "time_column": "", + "group_by": ["num"], + "aggregate": "Count", + "columns": [], + } + rv = client.post( + "/api/query", data=json.dumps(payload), content_type="application/json" + ) + data = rv.get_json() + assert rv.status_code == 200 + rows = sorted(data["rows"]) + assert rows == [[1, 1], [2, 1]] + assert all(len(r) == 2 for r in rows) + + def test_order_by_samples_table() -> None: app = server.app client = app.test_client() (scubaduck) root@5b69743466e4:/workspace/scubaduck#

The recent commit added a conditional to prevent count(*) from being labeled as "Count," but Hits column still automatically appears.

shell
(scubaduck) root@5b69743466e4:/workspace/scubaduck# git show --stat 80a8ced commit 80a8cedc3fc2d49185d00e1c7ed03f2cbb8cc8ab Author: Edward Z. Yang <ezyang@mit.edu> Date: Thu May 22 22:37:26 2025 -0400 fix(ui): show count series for numeric timestamps (#148) scubaduck/static/js/timeseries_chart.js | 8 ++++++- tests/test_server_errors.py | 2 +- tests/test_web_timeseries.py | 39 +++++++++++++++++++++++++++++++++ 3 files changed, 47 insertions(+), 2 deletions(-)
ad commit e90ea2e7d9288691773ea777ed5f4dffe4b5f6f3 Author: Edward Z. Yang <ezyang@mit.edu> Date: Thu May 22 22:37:08 2025 -0400 fix(server): json encode bytes (#149) scubaduck/server.py | 6 ++++++ tests/test_server_db_types.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+) (scubaduck) root@5b69743466e4:/workspace/scubaduck#

I’ll review earlier commits to find references to “show_hits” or “invalid column.”

shell
(scubaduck) root@5b69743466e4:/workspace/scubaduck# git log -p e08b86b^..HEAD | grep -n "Unknown column" -n (scubaduck) root@5b69743466e4:/workspace/scubaduck#

I can't replicate the bug with my current dataset, but I’ll try with another one.

shell
(scubaduck) root@5b69743466e4:/workspace/scubaduck# python - <<'EOF' > import json > from scubaduck import server > > app = server.create_app("TEST") > client = app.test_client() > payload = { > "table": "extra", > "graph_type": "table", > "time_column": "", > "group_by": ["desc"], > "order_by": "Samples", > "limit": 10, > } > rv = client.post("/api/query", data=json.dumps(payload), content_type="applica tion/json") > print('status', rv.status_code) > print('data', rv.get_json()) > EOF status 200 data {'rows': [['y', 1], ['x', 1]], 'sql': 'SELECT "desc", count(*) AS Hits\nFRO M "extra"\nGROUP BY "desc"\nORDER BY "Hits" ASC\nLIMIT 10'} (scubaduck) root@5b69743466e4:/workspace/scubaduck#

It seems the bug might be related to when "Show Hits" unchecked still sends "Samples" ordering.

shell
(scubaduck) root@5b69743466e4:/workspace/scubaduck# git show 9b4de028df02ef944ba e7bff88c2845de91abf0d:scubaduck/static/js/table_view.js | sed -n '100,150p' if (sortState.index !== idx) { sortState.index = idx; sortState.dir = "asc"; } else if (sortState.dir === "asc") { sortState.dir = "desc"; } else if (sortState.dir === "desc") { sortState.index = null; sortState.dir = null; } else { sortState.dir = "asc"; } let rows = originalRows.slice(); if (sortState.index !== null) { rows.sort((a, b) => { const va = a[sortState.index]; const vb = b[sortState.index]; if (va === vb) return 0; if (sortState.dir === "desc") return va < vb ? 1 : -1; return va > vb ? 1 : -1; }); } renderTable(rows); } function showResults(data) { window.lastResults = data; const hideHits = (graphTypeSel.value === "table" || graphTypeSel.value === "timeseries") && !document.getElementById("show_hits").checked; if (hideHits && data.rows.length) { const groupCount = (graphTypeSel.value === "timeseries" ? 1 : 0) + ((groupBy.chips || []).length || 0); data.rows.forEach((r) => r.splice(groupCount, 1)); } const view = document.getElementById("view"); if (graphTypeSel.value === "timeseries") { showTimeSeries(data); } else { if (data.rows.length === 0) { view.innerHTML = '<p id="empty-message">Empty data provided to table</p><table id="result s"></table>'; } else { view.innerHTML = '<table id="results"></table>'; } originalRows = data.rows.slice(); sortState = { index: null, dir: null }; renderTable(originalRows); } const sqlEl = document.createElement("pre"); sqlEl.id = "sql_query"; (scubaduck) root@5b69743466e4:/workspace/scubaduck#

Since I can't replicate the bug, I'll review server.py post-commit a8074f6 for potential issues.

shell
(scubaduck) root@5b69743466e4:/workspace/scubaduck# grep -n "valid_cols" -n scub aduck/server.py 562: valid_cols = set(column_types.keys()) 563: valid_cols.update(params.derived_columns.keys()) 564: valid_cols.add("Hits") 568: if params.x_axis is None or params.x_axis not in valid_cols: 587: if col not in valid_cols: 590: if col not in valid_cols: 592: if params.order_by and params.order_by not in valid_cols: (scubaduck) root@5b69743466e4:/workspace/scubaduck#

I’ll check around line 556 to confirm this section.

shell
(scubaduck) root@5b69743466e4:/workspace/scubaduck# sed -n '540,610p' scubaduck/ server.py return jsonify({"error": "Invalid table"}), 400 column_types = get_columns(params.table) if params.time_column and params.time_column not in column_types: return jsonify({"error": "Invalid time_column"}), 400 if params.time_unit not in {"s", "ms", "us", "ns"}: return jsonify({"error": "Invalid time_unit"}), 400 if params.graph_type not in {"table", "timeseries"} and ( params.group_by or params.aggregate or params.show_hits ): return ( jsonify( { "error": "group_by, aggregate and show_hits are only val id for table or timeseries view" } ), 400, ) valid_cols = set(column_types.keys()) valid_cols.update(params.derived_columns.keys()) valid_cols.add("Hits") if params.graph_type == "timeseries": if params.x_axis is None: params.x_axis = params.time_column if params.x_axis is None or params.x_axis not in valid_cols: return jsonify({"error": "Invalid x_axis"}), 400 ctype = column_types.get(params.x_axis, "").upper() is_time = any(t in ctype for t in ["TIMESTAMP", "DATE", "TIME"]) is_numeric = any( t in ctype for t in [ "INT", "DECIMAL", "REAL", "DOUBLE", "FLOAT", "NUMERIC", "HUGEINT", ] ) if not (is_time or is_numeric): return jsonify({"error": "x_axis must be a time column"}), 400 for col in params.columns: if col not in valid_cols: return jsonify({"error": f"Unknown column: {col}"}), 400 for col in params.group_by: if col not in valid_cols: return jsonify({"error": f"Unknown column: {col}"}), 400 if params.order_by and params.order_by not in valid_cols: return jsonify({"error": f"Unknown column: {params.order_by}"}), 400 if params.group_by or params.graph_type == "timeseries": agg = (params.aggregate or "count").lower() if agg.startswith("p") or agg == "sum": need_numeric = True allow_time = False elif agg == "avg" or agg in {"min", "max"}: need_numeric = False allow_time = True else: need_numeric = False allow_time = False if need_numeric or allow_time: for c in params.columns: if c in params.group_by or c == params.x_axis: continue if c not in column_types: (scubaduck) root@5b69743466e4:/workspace/scubaduck#

It seems like the front-end might omit "Hits" when show_hits is false, causing issues.

The issue might come from switching graph types and how "Samples" is handled in the query.

The mapping happens before valid_cols, so no bug found so far. I'll confirm the logic.

shell
(scubaduck) root@5b69743466e4:/workspace/scubaduck# sed -n '500,575p' scubaduck/ server.py [substr], ).fetchall() values = [r[0] for r in rows] _cache_set(key, values) return jsonify(values) @app.route("/api/query", methods=["POST"]) def query() -> Any: # pyright: ignore[reportUnusedFunction] payload = request.get_json(force=True) try: start = parse_time(payload.get("start")) end = parse_time(payload.get("end")) except Exception as exc: return jsonify({"error": str(exc)}), 400 params = QueryParams( start=start, end=end, order_by=payload.get("order_by"), order_dir=payload.get("order_dir", "ASC"), limit=payload.get("limit"), columns=payload.get("columns", []), derived_columns=payload.get("derived_columns", {}), graph_type=payload.get("graph_type", "samples"), group_by=payload.get("group_by", []), aggregate=payload.get("aggregate"), show_hits=payload.get("show_hits", False), x_axis=payload.get("x_axis"), granularity=payload.get("granularity", "Auto"), fill=payload.get("fill", "0"), table=payload.get("table", default_table), time_column=payload.get("time_column", "timestamp"), time_unit=payload.get("time_unit", "s"), ) if params.order_by == "Samples": params.order_by = "Hits" for f in payload.get("filters", []): params.filters.append(Filter(f["column"], f["op"], f.get("value"))) if params.table not in tables: return jsonify({"error": "Invalid table"}), 400 column_types = get_columns(params.table) if params.time_column and params.time_column not in column_types: return jsonify({"error": "Invalid time_column"}), 400 if params.time_unit not in {"s", "ms", "us", "ns"}: return jsonify({"error": "Invalid time_unit"}), 400 if params.graph_type not in {"table", "timeseries"} and ( params.group_by or params.aggregate or params.show_hits ): return ( jsonify( { "error": "group_by, aggregate and show_hits are only val id for table or timeseries view" } ), 400, ) valid_cols = set(column_types.keys()) valid_cols.update(params.derived_columns.keys()) valid_cols.add("Hits") if params.graph_type == "timeseries": if params.x_axis is None: params.x_axis = params.time_column if params.x_axis is None or params.x_axis not in valid_cols: return jsonify({"error": "Invalid x_axis"}), 400 ctype = column_types.get(params.x_axis, "").upper() is_time = any(t in ctype for t in ["TIMESTAMP", "DATE", "TIME"]) is_numeric = any( t in ctype for t in [ "INT",
(scubaduck) root@5b69743466e4:/workspace/scubaduck# grep -n "order_by" -n scubad uck/server.py 37: order_by: str | None = None 329: order_by = params.order_by 330: if order_by == "Samples": 331: order_by = "Hits" 332: order_by = order_by if order_by in selected_for_order else None 338: order_by=None, 352: if order_by: 353: lines.append(f"ORDER BY {_quote(order_by)} {params.order_dir}") 406: if order_by: 407: lines.append(f"ORDER BY {_quote(order_by)} {params.order_dir}") 518: order_by=payload.get("order_by"), 534: if params.order_by == "Samples": 535: params.order_by = "Hits" 592: if params.order_by and params.order_by not in valid_cols: 593: return jsonify({"error": f"Unknown column: {params.order_by}"}), 400 (scubaduck) root@5b69743466e4:/workspace/scubaduck#

The mapping happens at line 534, and valid_cols are computed after line 562. I'll check the tests to see how "order_by" works. Could there be interaction with the UI's "group_by"? Let's search for "order_by=Samples" in the tests.

shell
(scubaduck) root@5b69743466e4:/workspace/scubaduck# grep -n "Samples" tests/test _web_table.py (scubaduck) root@5b69743466e4:/workspace/scubaduck#

I'll add a test for the server bug, using "Sum" for aggregation and "user" for grouping.

shell
(scubaduck) root@5b69743466e4:/workspace/scubaduck# python - <<'EOF' > import json > from scubaduck import server > > app = server.app > client = app.test_client() > payload = { > "table": "events", > "start": "2024-01-01 00:00:00", > "end": "2024-01-03 00:00:00", > "graph_type": "table", > "group_by": ["user"], > "aggregate": "Sum", > "columns": ["value"], > "order_by": "Samples", > "order_dir": "DESC", > } > rv = client.post("/api/query", data=json.dumps(payload), content_type="applica tion/json") > print('status', rv.status_code) > print('data', rv.get_json()) > EOF status 200 data {'end': '2024-01-03 00:00:00', 'rows': [['alice', 2, 40], ['bob', 1, 20], [ 'charlie', 1, 40]], 'sql': 'SELECT "user", count(*) AS Hits, sum("value") AS "va lue"\nFROM "events"\nWHERE "timestamp" >= \'2024-01-01 00:00:00\' AND "timestamp " <= \'2024-01-03 00:00:00\'\nGROUP BY "user"\nORDER BY "Hits" DESC', 'start': ' 2024-01-01 00:00:00'} (scubaduck) root@5b69743466e4:/workspace/scubaduck#

The bug might happen when 'Samples' is sent mistakenly in columns or group_by.

The issue may arise from case-sensitivity or user-provided "Samples" in the query.

The bug could occur due to mismatches in the order of operations or environment setup.

The bug could arise when group_by is empty, show_hits true, and "Samples" is selected.

The bug may be due to Hits not being included in valid_cols without aggregation or group_by.

I'll check if server responds correctly with 200 or 400 when order_by "Samples" is used.

shell
(scubaduck) root@5b69743466e4:/workspace/scubaduck# python - <<'EOF' > import json > from scubaduck import server > > app = server.app > client = app.test_client() > payload = { > "table": "events", > "start": "2024-01-01 00:00:00", > "end": "2024-01-03 00:00:00", > "graph_type": "table", > "order_by": "Samples", > "order_dir": "DESC", > "limit": 10, > "columns": ["timestamp", "event", "value", "user"], > } > rv = client.post("/api/query", data=json.dumps(payload), content_type="applica tion/json") > print('status', rv.status_code) > print('data', rv.get_json()) > EOF status 200 data {'end': '2024-01-03 00:00:00', 'rows': [['Mon, 01 Jan 2024 00:00:00 GMT', ' login', 10, 'alice'], ['Mon, 01 Jan 2024 01:00:00 GMT', 'logout', 20, 'bob'], [' Tue, 02 Jan 2024 00:00:00 GMT', 'login', 30, 'alice'], ['Tue, 02 Jan 2024 03:00: 00 GMT', 'login', 40, 'charlie']], 'sql': 'SELECT "timestamp", "event", "value", "user"\nFROM "events"\nWHERE "timestamp" >= \'2024-01-01 00:00:00\' AND "timest amp" <= \'2024-01-03 00:00:00\'\nLIMIT 10', 'start': '2024-01-01 00:00:00'} (scubaduck) root@5b69743466e4:/workspace/scubaduck#

I'll handle potential discrepancies with "order_by" values, ensuring case insensitivity and stripped whitespace.

I’ll add a case-insensitive test for "samples" to check server behavior with lowercase input.

shell
(scubaduck) root@5b69743466e4:/workspace/scubaduck# python - <<'EOF' > import json > from scubaduck import server > > app = server.app > client = app.test_client() > payload = { > "table": "events", > "start": "2024-01-01 00:00:00", > "end": "2024-01-03 00:00:00", > "graph_type": "table", > "order_by": "samples", # lowercase > "order_dir": "DESC", > "limit": 10, > "columns": [], > "group_by": ["user"], > } > rv = client.post("/api/query", data=json.dumps(payload), content_type="applica tion/json") > print('status', rv.status_code) > print('data', rv.get_json()) > EOF status 400 data {'error': 'Unknown column: samples'} (scubaduck) root@5b69743466e4:/workspace/scubaduck#

I’ve found the bug happens when "Samples" is lowercase. I’ll adjust the query to handle case insensitivity and add a test for it. Time to modify server.py to fix the issue!

shell
(scubaduck) root@5b69743466e4:/workspace/scubaduck# apply_patch<<'PATCH' > *** Begin Patch > *** Update File: scubaduck/server.py > @@ > - if params.order_by == "Samples": > - params.order_by = "Hits" > + if params.order_by and params.order_by.strip().lower() == "samples": > + params.order_by = "Hits" > *** End Patch > PATCH Success. Updated the following files: M scubaduck/server.py (scubaduck) root@5b69743466e4:/workspace/scubaduck#

Time to revise the build_query function. Let's get that updated!

shell
(scubaduck) root@5b69743466e4:/workspace/scubaduck# grep -n "order_by = params.o rder_by" -n scubaduck/server.py 329: order_by = params.order_by
(scubaduck) root@5b69743466e4:/workspace/scubaduck# sed -n '324,336p' scubaduck/ server.py selected_for_order.add("Hits") else: select_parts.extend(_quote(c) for c in params.columns) selected_for_order.update(params.columns) order_by = params.order_by if order_by == "Samples": order_by = "Hits" order_by = order_by if order_by in selected_for_order else None if has_agg and params.derived_columns: inner_params = replace( params,
(scubaduck) root@5b69743466e4:/workspace/scubaduck# apply_patch<<'PATCH' > *** Begin Patch > *** Update File: scubaduck/server.py > @@ > - order_by = params.order_by > - if order_by == "Samples": > - order_by = "Hits" > + order_by = params.order_by > + if order_by and str(order_by).strip().lower() == "samples": > + order_by = "Hits" > order_by = order_by if order_by in selected_for_order else None > *** End Patch > PATCH Success. Updated the following files: M scubaduck/server.py (scubaduck) root@5b69743466e4:/workspace/scubaduck#

I’ll add the new test for case-insensitive mapping to test_server_timeseries.py.

shell
(scubaduck) root@5b69743466e4:/workspace/scubaduck# grep -n "order_by_samples" - n tests/test_server_timeseries.py 316:def test_order_by_samples_table() -> None: 338:def test_order_by_samples_timeseries() -> None:
_server_timeseries.py def test_order_by_samples_table() -> None: app = server.app client = app.test_client() payload: dict[str, Any] = { "table": "events", "start": "2024-01-01 00:00:00", "end": "2024-01-03 00:00:00", "graph_type": "table", "order_by": "Samples", "order_dir": "DESC", "limit": 10, "columns": [], "group_by": ["user"], } rv = client.post( "/api/query", data=json.dumps(payload), content_type="application/json" ) data = rv.get_json() assert rv.status_code == 200 assert 'ORDER BY "Hits" DESC' in data["sql"] def test_order_by_samples_timeseries() -> None: app = server.app client = app.test_client() payload: dict[str, Any] = { "table": "events", "start": "2024-01-01 00:00:00", "end": "2024-01-03 00:00:00", "graph_type": "timeseries", "order_by": "Samples", "order_dir": "DESC", "x_axis": "timestamp", "granularity": "1 day", "columns": [], } rv = client.post( "/api/query", data=json.dumps(payload), content_type="application/json" ) data = rv.get_json() assert rv.status_code == 200 assert 'ORDER BY "Hits" DESC' in data["sql"] def test_show_hits_client_side() -> None: (scubaduck) root@5b69743466e4:/workspace/scubaduck#

I'll insert the new test after "test_order_by_samples_timeseries" or just past it.

shell
> *** Update File: tests/test_server_timeseries.py > @@ > def test_order_by_samples_timeseries() -> None: > @@ > assert 'ORDER BY "Hits" DESC' in data["sql"] > + > + > +def test_order_by_samples_case_insensitive() -> None: > + app = server.app > + client = app.test_client() > + payload: dict[str, Any] = { > + "table": "events", > + "start": "2024-01-01 00:00:00", > + "end": "2024-01-03 00:00:00", > + "graph_type": "table", > + "order_by": "samples", > + "order_dir": "DESC", > + "limit": 10, > + "columns": [], > + "group_by": ["user"], > + } > + rv = client.post( > + "/api/query", data=json.dumps(payload), content_type="application/jso n" > + ) > + data = rv.get_json() > + assert rv.status_code == 200 > + assert 'ORDER BY "Hits" DESC' in data["sql"] > *** End Patch > PATCH Success. Updated the following files: M tests/test_server_timeseries.py
(scubaduck) root@5b69743466e4:/workspace/scubaduck# ruff format scubaduck/server .py tests/test_server_timeseries.py 2 files left unchanged
(scubaduck) root@5b69743466e4:/workspace/scubaduck# ruff check scubaduck/server. py tests/test_server_timeseries.py All checks passed! (scubaduck) root@5b69743466e4:/workspace/scubaduck#

I'll go ahead and run pyright to check for any type issues.

shell
(scubaduck) root@5b69743466e4:/workspace/scubaduck# pyright scubaduck/server.py tests/test_server_timeseries.py > /tmp/pyright.log && tail -n 20 /tmp/pyright.lo g
0 errors, 0 warnings, 0 informations (scubaduck) root@5b69743466e4:/workspace/scubaduck#

I’ll run the full suite of tests, as per AGENTS’ instructions, using pytest -q.

shell
(scubaduck) root@5b69743466e4:/workspace/scubaduck# pytest -q
bringing up nodes...






........................................................................ [ 56%]

........................................................ [100%] 128 passed in 16.34s (scubaduck) root@5b69743466e4:/workspace/scubaduck#

Tests are good! Time to go ahead and commit the changes now.

shell
(scubaduck) root@5b69743466e4:/workspace/scubaduck# git status --short M scubaduck/server.py M tests/test_server_timeseries.py (scubaduck) root@5b69743466e4:/workspace/scubaduck#

I'll run ruff over the entire repo once more to check everything.

shell
(scubaduck) root@5b69743466e4:/workspace/scubaduck# ruff format scubaduck/server .py tests/test_server_timeseries.py 2 files left unchanged
(scubaduck) root@5b69743466e4:/workspace/scubaduck# ruff check scubaduck/server. py tests/test_server_timeseries.py All checks passed!
(scubaduck) root@5b69743466e4:/workspace/scubaduck# pyright scubaduck/server.py tests/test_server_timeseries.py > /tmp/pyright.log && tail -n 20 /tmp/pyright.lo g
0 errors, 0 warnings, 0 informations (scubaduck) root@5b69743466e4:/workspace/scubaduck#

I'll run git diff to check the changes made in the code.

shell
(scubaduck) root@5b69743466e4:/workspace/scubaduck# git diff --stat scubaduck/server.py | 4 ++-- tests/test_server_timeseries.py | 22 ++++++++++++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-)
(scubaduck) root@5b69743466e4:/workspace/scubaduck# git add scubaduck/server.py tests/test_server_timeseries.py
(scubaduck) root@5b69743466e4:/workspace/scubaduck# git commit -m "fix(server): handle lowercase samples order_by" && git status --short [work c466220] fix(server): handle lowercase samples order_by 2 files changed, 24 insertions(+), 2 deletions(-)
(scubaduck) root@5b69743466e4:/workspace/scubaduck# git status --short (scubaduck) root@5b69743466e4:/workspace/scubaduck#

I'll craft the PR message, referencing the relevant code lines from server.py.

shell
(scubaduck) root@5b69743466e4:/workspace/scubaduck# nl -ba scubaduck/server.py | sed -n '522,538p' 522 derived_columns=payload.get("derived_columns", {}), 523 graph_type=payload.get("graph_type", "samples"), 524 group_by=payload.get("group_by", []), 525 aggregate=payload.get("aggregate"), 526 show_hits=payload.get("show_hits", False), 527 x_axis=payload.get("x_axis"), 528 granularity=payload.get("granularity", "Auto"), 529 fill=payload.get("fill", "0"), 530 table=payload.get("table", default_table), 531 time_column=payload.get("time_column", "timestamp"), 532 time_unit=payload.get("time_unit", "s"), 533 ) 534 if params.order_by and params.order_by.strip().lower() == "sampl es": 535 params.order_by = "Hits" 536 for f in payload.get("filters", []): 537 params.filters.append(Filter(f["column"], f["op"], f.get("va lue"))) 538
(scubaduck) root@5b69743466e4:/workspace/scubaduck# nl -ba scubaduck/server.py | sed -n '324,336p' 324 selected_for_order.add("Hits") 325 else: 326 select_parts.extend(_quote(c) for c in params.columns) 327 selected_for_order.update(params.columns) 328 329 order_by = params.order_by 330 if order_by and str(order_by).strip().lower() == "samples": 331 order_by = "Hits" 332 order_by = order_by if order_by in selected_for_order else None 333 334 if has_agg and params.derived_columns: 335 inner_params = replace( 336 params,
(scubaduck) root@5b69743466e4:/workspace/scubaduck# nl -ba tests/test_server_tim eseries.py | sed -n '334,380p' 334 assert rv.status_code == 200 335 assert 'ORDER BY "Hits" DESC' in data["sql"] 336 337 338 def test_order_by_samples_timeseries() -> None: 339 app = server.app 340 client = app.test_client() 341 payload: dict[str, Any] = { 342 "table": "events", 343 "start": "2024-01-01 00:00:00", 344 "end": "2024-01-03 00:00:00", 345 "graph_type": "timeseries", 346 "order_by": "Samples", 347 "order_dir": "DESC", 348 "x_axis": "timestamp", 349 "granularity": "1 day", 350 "columns": [], 351 } 352 rv = client.post( 353 "/api/query", data=json.dumps(payload), content_type="applicatio n/json" 354 ) 355 data = rv.get_json() 356 assert rv.status_code == 200 357 assert 'ORDER BY "Hits" DESC' in data["sql"] 358 359 360 def test_order_by_samples_case_insensitive() -> None: 361 app = server.app 362 client = app.test_client() 363 payload: dict[str, Any] = { 364 "table": "events", 365 "start": "2024-01-01 00:00:00", 366 "end": "2024-01-03 00:00:00", 367 "graph_type": "table", 368 "order_by": "samples", 369 "order_dir": "DESC", 370 "limit": 10, 371 "columns": [], 372 "group_by": ["user"], 373 } 374 rv = client.post( 375 "/api/query", data=json.dumps(payload), content_type="applicatio n/json" 376 ) 377 data = rv.get_json() 378 assert rv.status_code == 200 379 assert 'ORDER BY "Hits" DESC' in data["sql"] 380 (scubaduck) root@5b69743466e4:/workspace/scubaduck#

Time to write the PR message with the relevant details and changes.