Task ID: task_e_682f6eb2500483239b34f828779ebee3
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (5/33)
pluggy ------------------------------ 0 B/20.06 KiB
execnet ------------------------------ 14.88 KiB/39.66 KiB
requests ------------------------------ 48.00 KiB/63.41 KiB
flask ------------------------------ 14.88 KiB/100.88 KiB
jinja2 ------------------------------ 0 B/131.74 KiB
charset-normalizer ------------------------------ 16.00 KiB/145.08 KiB
werkzeug ------------------------------ 141.22 KiB/219.24 KiB
python-dateutil ------------------------------ 56.52 KiB/224.50 KiB
greenlet ------------------------------ 461.24 KiB/589.71 KiB
pyright ------------------------------ 462.99 KiB/5.31 MiB
ruff ------------------------------ 91.98 KiB/11.02 MiB
duckdb ------------------------------ 40.57 KiB/19.27 MiB
playwright ------------------------------ 32.00 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (5/33)
pluggy ------------------------------ 0 B/20.06 KiB
execnet ------------------------------ 14.88 KiB/39.66 KiB
flask ------------------------------ 14.88 KiB/100.88 KiB
jinja2 ------------------------------ 0 B/131.74 KiB
charset-normalizer ------------------------------ 16.00 KiB/145.08 KiB
werkzeug ------------------------------ 157.22 KiB/219.24 KiB
python-dateutil ------------------------------ 56.52 KiB/224.50 KiB
greenlet ------------------------------ 461.24 KiB/589.71 KiB
pyright ------------------------------ 542.99 KiB/5.31 MiB
ruff ------------------------------ 91.98 KiB/11.02 MiB
duckdb ------------------------------ 40.57 KiB/19.27 MiB
playwright ------------------------------ 32.00 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (5/33)
pytest-base-url ------------------------------ 5.18 KiB/5.18 KiB
pluggy ------------------------------ 8.00 KiB/20.06 KiB
execnet ------------------------------ 30.88 KiB/39.66 KiB
typing-extensions ------------------------------ 14.86 KiB/44.73 KiB
idna ------------------------------ 6.87 KiB/68.79 KiB
click ------------------------------ 6.87 KiB/99.76 KiB
flask ------------------------------ 30.88 KiB/100.88 KiB
jinja2 ------------------------------ 8.00 KiB/131.74 KiB
charset-normalizer ------------------------------ 32.00 KiB/145.08 KiB
werkzeug ------------------------------ 157.22 KiB/219.24 KiB
python-dateutil ------------------------------ 72.52 KiB/224.50 KiB
pytest ------------------------------ 6.87 KiB/335.58 KiB
greenlet ------------------------------ 477.24 KiB/589.71 KiB
pyright ------------------------------ 622.99 KiB/5.31 MiB
ruff ------------------------------ 139.98 KiB/11.02 MiB
duckdb ------------------------------ 88.57 KiB/19.27 MiB
playwright ------------------------------ 48.00 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (5/33)
pytest-base-url ------------------------------ 5.18 KiB/5.18 KiB
itsdangerous ------------------------------ 8.00 KiB/15.85 KiB
pluggy ------------------------------ 16.00 KiB/20.06 KiB
markupsafe ------------------------------ 6.87 KiB/22.58 KiB
execnet ------------------------------ 30.88 KiB/39.66 KiB
idna ------------------------------ 32.00 KiB/68.79 KiB
text-unidecode ------------------------------ 8.00 KiB/76.32 KiB
click ------------------------------ 46.88 KiB/99.76 KiB
flask ------------------------------ 62.88 KiB/100.88 KiB
jinja2 ------------------------------ 32.00 KiB/131.74 KiB
charset-normalizer ------------------------------ 126.96 KiB/145.08 KiB
werkzeug ------------------------------ 189.22 KiB/219.24 KiB
python-dateutil ------------------------------ 216.52 KiB/224.50 KiB
pytest ------------------------------ 30.88 KiB/335.58 KiB
greenlet ------------------------------ 477.24 KiB/589.71 KiB
pyright ------------------------------ 814.99 KiB/5.31 MiB
ruff ------------------------------ 331.98 KiB/11.02 MiB
duckdb ------------------------------ 280.57 KiB/19.27 MiB
playwright ------------------------------ 48.00 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (5/33)
itsdangerous ------------------------------ 8.00 KiB/15.85 KiB
pluggy ------------------------------ 16.00 KiB/20.06 KiB
markupsafe ------------------------------ 6.87 KiB/22.58 KiB
execnet ------------------------------ 30.88 KiB/39.66 KiB
idna ------------------------------ 32.00 KiB/68.79 KiB
text-unidecode ------------------------------ 8.00 KiB/76.32 KiB
click ------------------------------ 62.88 KiB/99.76 KiB
flask ------------------------------ 62.88 KiB/100.88 KiB
jinja2 ------------------------------ 32.00 KiB/131.74 KiB
charset-normalizer ------------------------------ 126.96 KiB/145.08 KiB
werkzeug ------------------------------ 189.22 KiB/219.24 KiB
python-dateutil ------------------------------ 216.52 KiB/224.50 KiB
pytest ------------------------------ 30.88 KiB/335.58 KiB
greenlet ------------------------------ 477.24 KiB/589.71 KiB
pyright ------------------------------ 814.99 KiB/5.31 MiB
ruff ------------------------------ 331.98 KiB/11.02 MiB
duckdb ------------------------------ 296.57 KiB/19.27 MiB
playwright ------------------------------ 48.00 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (5/33)
itsdangerous ------------------------------ 8.00 KiB/15.85 KiB
pluggy ------------------------------ 20.06 KiB/20.06 KiB
markupsafe ------------------------------ 22.58 KiB/22.58 KiB
execnet ------------------------------ 39.66 KiB/39.66 KiB
idna ------------------------------ 64.00 KiB/68.79 KiB
text-unidecode ------------------------------ 76.32 KiB/76.32 KiB
click ------------------------------ 62.88 KiB/99.76 KiB
flask ------------------------------ 62.88 KiB/100.88 KiB
jinja2 ------------------------------ 60.36 KiB/131.74 KiB
charset-normalizer ------------------------------ 126.96 KiB/145.08 KiB
werkzeug ------------------------------ 205.22 KiB/219.24 KiB
python-dateutil ------------------------------ 224.50 KiB/224.50 KiB
pytest ------------------------------ 46.88 KiB/335.58 KiB
greenlet ------------------------------ 477.24 KiB/589.71 KiB
pyright ------------------------------ 910.99 KiB/5.31 MiB
ruff ------------------------------ 427.98 KiB/11.02 MiB
duckdb ------------------------------ 376.57 KiB/19.27 MiB
playwright ------------------------------ 64.00 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (5/33)
itsdangerous ------------------------------ 15.85 KiB/15.85 KiB
pluggy ------------------------------ 20.06 KiB/20.06 KiB
markupsafe ------------------------------ 22.58 KiB/22.58 KiB
execnet ------------------------------ 39.66 KiB/39.66 KiB
text-unidecode ------------------------------ 76.32 KiB/76.32 KiB
click ------------------------------ 78.88 KiB/99.76 KiB
flask ------------------------------ 78.88 KiB/100.88 KiB
jinja2 ------------------------------ 76.36 KiB/131.74 KiB
charset-normalizer ------------------------------ 142.96 KiB/145.08 KiB
werkzeug ------------------------------ 219.24 KiB/219.24 KiB
python-dateutil ------------------------------ 224.50 KiB/224.50 KiB
pytest ------------------------------ 76.53 KiB/335.58 KiB
greenlet ------------------------------ 493.24 KiB/589.71 KiB
pyright ------------------------------ 990.99 KiB/5.31 MiB
ruff ------------------------------ 495.98 KiB/11.02 MiB
duckdb ------------------------------ 462.91 KiB/19.27 MiB
playwright ------------------------------ 64.00 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (5/33)
itsdangerous ------------------------------ 15.85 KiB/15.85 KiB
pluggy ------------------------------ 20.06 KiB/20.06 KiB
markupsafe ------------------------------ 22.58 KiB/22.58 KiB
execnet ------------------------------ 39.66 KiB/39.66 KiB
text-unidecode ------------------------------ 76.32 KiB/76.32 KiB
click ------------------------------ 78.88 KiB/99.76 KiB
flask ------------------------------ 78.88 KiB/100.88 KiB
jinja2 ------------------------------ 92.36 KiB/131.74 KiB
charset-normalizer ------------------------------ 142.96 KiB/145.08 KiB
werkzeug ------------------------------ 219.24 KiB/219.24 KiB
pytest ------------------------------ 76.53 KiB/335.58 KiB
greenlet ------------------------------ 493.24 KiB/589.71 KiB
pyright ------------------------------ 1006.99 KiB/5.31 MiB
ruff ------------------------------ 511.98 KiB/11.02 MiB
duckdb ------------------------------ 478.91 KiB/19.27 MiB
playwright ------------------------------ 64.00 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (5/33)
itsdangerous ------------------------------ 15.85 KiB/15.85 KiB
markupsafe ------------------------------ 22.58 KiB/22.58 KiB
execnet ------------------------------ 39.66 KiB/39.66 KiB
text-unidecode ------------------------------ 76.32 KiB/76.32 KiB
click ------------------------------ 78.88 KiB/99.76 KiB
flask ------------------------------ 78.88 KiB/100.88 KiB
jinja2 ------------------------------ 92.36 KiB/131.74 KiB
charset-normalizer ------------------------------ 142.96 KiB/145.08 KiB
werkzeug ------------------------------ 219.24 KiB/219.24 KiB
pytest ------------------------------ 76.53 KiB/335.58 KiB
greenlet ------------------------------ 493.24 KiB/589.71 KiB
pyright ------------------------------ 1022.99 KiB/5.31 MiB
ruff ------------------------------ 527.98 KiB/11.02 MiB
duckdb ------------------------------ 478.91 KiB/19.27 MiB
playwright ------------------------------ 64.00 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (5/33)
itsdangerous ------------------------------ 15.85 KiB/15.85 KiB
execnet ------------------------------ 39.66 KiB/39.66 KiB
text-unidecode ------------------------------ 76.32 KiB/76.32 KiB
click ------------------------------ 94.88 KiB/99.76 KiB
flask ------------------------------ 78.88 KiB/100.88 KiB
jinja2 ------------------------------ 92.36 KiB/131.74 KiB
charset-normalizer ------------------------------ 142.96 KiB/145.08 KiB
werkzeug ------------------------------ 219.24 KiB/219.24 KiB
pytest ------------------------------ 76.53 KiB/335.58 KiB
greenlet ------------------------------ 493.24 KiB/589.71 KiB
pyright ------------------------------ 1.01 MiB/5.31 MiB
ruff ------------------------------ 543.98 KiB/11.02 MiB
duckdb ------------------------------ 510.91 KiB/19.27 MiB
playwright ------------------------------ 80.00 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (5/33)
itsdangerous ------------------------------ 15.85 KiB/15.85 KiB
text-unidecode ------------------------------ 76.32 KiB/76.32 KiB
click ------------------------------ 99.76 KiB/99.76 KiB
flask ------------------------------ 78.88 KiB/100.88 KiB
jinja2 ------------------------------ 108.36 KiB/131.74 KiB
charset-normalizer ------------------------------ 145.08 KiB/145.08 KiB
werkzeug ------------------------------ 219.24 KiB/219.24 KiB
pytest ------------------------------ 108.53 KiB/335.58 KiB
greenlet ------------------------------ 493.24 KiB/589.71 KiB
pyright ------------------------------ 1.06 MiB/5.31 MiB
ruff ------------------------------ 591.98 KiB/11.02 MiB
duckdb ------------------------------ 558.91 KiB/19.27 MiB
playwright ------------------------------ 80.00 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (5/33)
text-unidecode ------------------------------ 76.32 KiB/76.32 KiB
click ------------------------------ 99.76 KiB/99.76 KiB
flask ------------------------------ 78.88 KiB/100.88 KiB
jinja2 ------------------------------ 108.36 KiB/131.74 KiB
charset-normalizer ------------------------------ 145.08 KiB/145.08 KiB
werkzeug ------------------------------ 219.24 KiB/219.24 KiB
pytest ------------------------------ 108.53 KiB/335.58 KiB
greenlet ------------------------------ 493.24 KiB/589.71 KiB
pyright ------------------------------ 1.09 MiB/5.31 MiB
ruff ------------------------------ 623.98 KiB/11.02 MiB
duckdb ------------------------------ 590.91 KiB/19.27 MiB
playwright ------------------------------ 80.00 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (5/33)
click ------------------------------ 99.76 KiB/99.76 KiB
flask ------------------------------ 78.88 KiB/100.88 KiB
jinja2 ------------------------------ 108.36 KiB/131.74 KiB
charset-normalizer ------------------------------ 145.08 KiB/145.08 KiB
werkzeug ------------------------------ 219.24 KiB/219.24 KiB
pytest ------------------------------ 108.53 KiB/335.58 KiB
greenlet ------------------------------ 493.24 KiB/589.71 KiB
pyright ------------------------------ 1.09 MiB/5.31 MiB
ruff ------------------------------ 623.98 KiB/11.02 MiB
duckdb ------------------------------ 590.91 KiB/19.27 MiB
playwright ------------------------------ 80.00 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (5/33)
click ------------------------------ 99.76 KiB/99.76 KiB
flask ------------------------------ 78.88 KiB/100.88 KiB
jinja2 ------------------------------ 108.36 KiB/131.74 KiB
charset-normalizer ------------------------------ 145.08 KiB/145.08 KiB
pytest ------------------------------ 108.53 KiB/335.58 KiB
greenlet ------------------------------ 493.24 KiB/589.71 KiB
pyright ------------------------------ 1.11 MiB/5.31 MiB
ruff ------------------------------ 639.98 KiB/11.02 MiB
duckdb ------------------------------ 590.91 KiB/19.27 MiB
playwright ------------------------------ 96.00 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (5/33)
click ------------------------------ 99.76 KiB/99.76 KiB
flask ------------------------------ 94.88 KiB/100.88 KiB
jinja2 ------------------------------ 124.36 KiB/131.74 KiB
pytest ------------------------------ 108.53 KiB/335.58 KiB
greenlet ------------------------------ 493.24 KiB/589.71 KiB
pyright ------------------------------ 1.12 MiB/5.31 MiB
ruff ------------------------------ 655.98 KiB/11.02 MiB
duckdb ------------------------------ 622.91 KiB/19.27 MiB
playwright ------------------------------ 96.00 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (5/33)
flask ------------------------------ 100.88 KiB/100.88 KiB
jinja2 ------------------------------ 131.74 KiB/131.74 KiB
pytest ------------------------------ 124.53 KiB/335.58 KiB
greenlet ------------------------------ 493.24 KiB/589.71 KiB
pyright ------------------------------ 1.17 MiB/5.31 MiB
ruff ------------------------------ 703.98 KiB/11.02 MiB
duckdb ------------------------------ 670.91 KiB/19.27 MiB
playwright ------------------------------ 96.00 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (5/33)
flask ------------------------------ 100.88 KiB/100.88 KiB
jinja2 ------------------------------ 131.74 KiB/131.74 KiB
pytest ------------------------------ 172.53 KiB/335.58 KiB
greenlet ------------------------------ 509.24 KiB/589.71 KiB
pyright ------------------------------ 1.31 MiB/5.31 MiB
ruff ------------------------------ 847.98 KiB/11.02 MiB
duckdb ------------------------------ 814.91 KiB/19.27 MiB
playwright ------------------------------ 96.00 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (5/33)
jinja2 ------------------------------ 131.74 KiB/131.74 KiB
pytest ------------------------------ 172.53 KiB/335.58 KiB
greenlet ------------------------------ 509.24 KiB/589.71 KiB
pyright ------------------------------ 1.33 MiB/5.31 MiB
ruff ------------------------------ 863.98 KiB/11.02 MiB
duckdb ------------------------------ 830.91 KiB/19.27 MiB
playwright ------------------------------ 112.00 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (5/33)
pytest ------------------------------ 188.53 KiB/335.58 KiB
greenlet ------------------------------ 509.24 KiB/589.71 KiB
pyright ------------------------------ 1.33 MiB/5.31 MiB
ruff ------------------------------ 927.98 KiB/11.02 MiB
duckdb ------------------------------ 894.91 KiB/19.27 MiB
playwright ------------------------------ 112.00 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (5/33)
pytest ------------------------------ 220.53 KiB/335.58 KiB
greenlet ------------------------------ 557.24 KiB/589.71 KiB
pyright ------------------------------ 1.67 MiB/5.31 MiB
ruff ------------------------------ 1.33 MiB/11.02 MiB
duckdb ------------------------------ 1.31 MiB/19.27 MiB
playwright ------------------------------ 272.00 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠸ Preparing packages... (26/33)
pytest ------------------------------ 316.53 KiB/335.58 KiB
greenlet ------------------------------ 589.71 KiB/589.71 KiB
pyright ------------------------------ 1.79 MiB/5.31 MiB
ruff ------------------------------ 1.89 MiB/11.02 MiB
duckdb ------------------------------ 1.87 MiB/19.27 MiB
playwright ------------------------------ 830.20 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠸ Preparing packages... (26/33)
pytest ------------------------------ 316.53 KiB/335.58 KiB
pyright ------------------------------ 1.79 MiB/5.31 MiB
ruff ------------------------------ 1.94 MiB/11.02 MiB
duckdb ------------------------------ 1.93 MiB/19.27 MiB
playwright ------------------------------ 894.20 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠸ Preparing packages... (26/33)
pyright ------------------------------ 1.81 MiB/5.31 MiB
ruff ------------------------------ 2.34 MiB/11.02 MiB
duckdb ------------------------------ 2.34 MiB/19.27 MiB
playwright ------------------------------ 1.29 MiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠸ Preparing packages... (26/33)
pyright ------------------------------ 1.83 MiB/5.31 MiB
ruff ------------------------------ 2.61 MiB/11.02 MiB
duckdb ------------------------------ 2.61 MiB/19.27 MiB
playwright ------------------------------ 1.56 MiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠸ Preparing packages... (26/33)
pyright ------------------------------ 1.84 MiB/5.31 MiB
ruff ------------------------------ 3.33 MiB/11.02 MiB
duckdb ------------------------------ 3.34 MiB/19.27 MiB
playwright ------------------------------ 2.27 MiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠸ Preparing packages... (26/33)
pyright ------------------------------ 1.89 MiB/5.31 MiB
ruff ------------------------------ 4.01 MiB/11.02 MiB
duckdb ------------------------------ 4.06 MiB/19.27 MiB
playwright ------------------------------ 2.97 MiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠼ Preparing packages... (28/33)
pyright ------------------------------ 1.92 MiB/5.31 MiB
ruff ------------------------------ 4.65 MiB/11.02 MiB
duckdb ------------------------------ 4.70 MiB/19.27 MiB
playwright ------------------------------ 3.64 MiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠼ Preparing packages... (28/33)
pyright ------------------------------ 1.95 MiB/5.31 MiB
ruff ------------------------------ 5.42 MiB/11.02 MiB
duckdb ------------------------------ 5.45 MiB/19.27 MiB
playwright ------------------------------ 4.36 MiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠼ Preparing packages... (28/33)
pyright ------------------------------ 1.98 MiB/5.31 MiB
ruff ------------------------------ 6.03 MiB/11.02 MiB
duckdb ------------------------------ 6.05 MiB/19.27 MiB
playwright ------------------------------ 4.98 MiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠼ Preparing packages... (28/33)
pyright ------------------------------ 2.02 MiB/5.31 MiB
ruff ------------------------------ 6.84 MiB/11.02 MiB
duckdb ------------------------------ 6.90 MiB/19.27 MiB
playwright ------------------------------ 5.83 MiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠴ Preparing packages... (28/33)
pyright ------------------------------ 2.08 MiB/5.31 MiB
ruff ------------------------------ 7.62 MiB/11.02 MiB
duckdb ------------------------------ 7.67 MiB/19.27 MiB
playwright ------------------------------ 6.58 MiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠴ Preparing packages... (28/33)
pyright ------------------------------ 2.09 MiB/5.31 MiB
ruff ------------------------------ 8.31 MiB/11.02 MiB
duckdb ------------------------------ 8.36 MiB/19.27 MiB
playwright ------------------------------ 7.27 MiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠴ Preparing packages... (28/33)
pyright ------------------------------ 2.12 MiB/5.31 MiB
ruff ------------------------------ 9.10 MiB/11.02 MiB
duckdb ------------------------------ 9.17 MiB/19.27 MiB
playwright ------------------------------ 8.08 MiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠴ Preparing packages... (28/33)
pyright ------------------------------ 2.14 MiB/5.31 MiB
ruff ------------------------------ 9.92 MiB/11.02 MiB
duckdb ------------------------------ 9.96 MiB/19.27 MiB
playwright ------------------------------ 8.89 MiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠦ Preparing packages... (28/33)
pyright ------------------------------ 2.16 MiB/5.31 MiB
ruff ------------------------------ 10.76 MiB/11.02 MiB
duckdb ------------------------------ 10.82 MiB/19.27 MiB
playwright ------------------------------ 9.72 MiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠦ Preparing packages... (28/33)
pyright ------------------------------ 2.17 MiB/5.31 MiB
duckdb ------------------------------ 11.15 MiB/19.27 MiB
playwright ------------------------------ 10.06 MiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠦ Preparing packages... (28/33)
pyright ------------------------------ 2.19 MiB/5.31 MiB
duckdb ------------------------------ 11.87 MiB/19.27 MiB
playwright ------------------------------ 10.81 MiB/43.05 MiB
Built scubaduck @ file:///workspace/scubaduck
⠦ Preparing packages... (28/33)
pyright ------------------------------ 2.19 MiB/5.31 MiB
duckdb ------------------------------ 12.39 MiB/19.27 MiB
playwright ------------------------------ 11.30 MiB/43.05 MiB
⠦ Preparing packages... (28/33)
pyright ------------------------------ 2.20 MiB/5.31 MiB
duckdb ------------------------------ 12.95 MiB/19.27 MiB
playwright ------------------------------ 11.86 MiB/43.05 MiB
⠦ Preparing packages... (28/33)
pyright ------------------------------ 2.22 MiB/5.31 MiB
duckdb ------------------------------ 14.25 MiB/19.27 MiB
playwright ------------------------------ 13.19 MiB/43.05 MiB
⠧ Preparing packages... (30/33)
pyright ------------------------------ 2.23 MiB/5.31 MiB
duckdb ------------------------------ 15.39 MiB/19.27 MiB
playwright ------------------------------ 14.33 MiB/43.05 MiB
⠧ Preparing packages... (30/33)
pyright ------------------------------ 2.25 MiB/5.31 MiB
duckdb ------------------------------ 16.64 MiB/19.27 MiB
playwright ------------------------------ 15.54 MiB/43.05 MiB
⠧ Preparing packages... (30/33)
pyright ------------------------------ 2.27 MiB/5.31 MiB
duckdb ------------------------------ 17.83 MiB/19.27 MiB
playwright ------------------------------ 16.75 MiB/43.05 MiB
⠧ Preparing packages... (30/33)
pyright ------------------------------ 2.30 MiB/5.31 MiB
duckdb ------------------------------ 18.95 MiB/19.27 MiB
playwright ------------------------------ 17.84 MiB/43.05 MiB
⠇ Preparing packages... (30/33)
pyright ------------------------------ 2.37 MiB/5.31 MiB
duckdb ------------------------------ 19.27 MiB/19.27 MiB
playwright ------------------------------ 19.70 MiB/43.05 MiB
⠇ Preparing packages... (30/33)
pyright ------------------------------ 2.37 MiB/5.31 MiB
playwright ------------------------------ 19.87 MiB/43.05 MiB
⠇ Preparing packages... (30/33)
pyright ------------------------------ 2.41 MiB/5.31 MiB
playwright ------------------------------ 22.12 MiB/43.05 MiB
⠇ Preparing packages... (30/33)
pyright ------------------------------ 2.42 MiB/5.31 MiB
playwright ------------------------------ 24.70 MiB/43.05 MiB
⠇ Preparing packages... (30/33)
pyright ------------------------------ 2.45 MiB/5.31 MiB
playwright ------------------------------ 27.30 MiB/43.05 MiB
⠋ Preparing packages... (31/33)
pyright ------------------------------ 2.47 MiB/5.31 MiB
playwright ------------------------------ 30.20 MiB/43.05 MiB
⠋ Preparing packages... (31/33)
pyright ------------------------------ 2.52 MiB/5.31 MiB
playwright ------------------------------ 32.78 MiB/43.05 MiB
⠋ Preparing packages... (31/33)
pyright ------------------------------ 2.53 MiB/5.31 MiB
playwright ------------------------------ 35.39 MiB/43.05 MiB
⠋ Preparing packages... (31/33)
pyright ------------------------------ 2.58 MiB/5.31 MiB
playwright ------------------------------ 37.83 MiB/43.05 MiB
⠙ Preparing packages... (31/33)
pyright ------------------------------ 2.59 MiB/5.31 MiB
playwright ------------------------------ 40.08 MiB/43.05 MiB
⠙ Preparing packages... (31/33)
pyright ------------------------------ 2.66 MiB/5.31 MiB
playwright ------------------------------ 41.12 MiB/43.05 MiB
⠙ Preparing packages... (31/33)
pyright ------------------------------ 2.75 MiB/5.31 MiB
playwright ------------------------------ 41.62 MiB/43.05 MiB
⠙ Preparing packages... (31/33)
pyright ------------------------------ 2.78 MiB/5.31 MiB
⠹ Preparing packages... (32/33)
pyright ------------------------------ 2.80 MiB/5.31 MiB
⠹ Preparing packages... (32/33)
pyright ------------------------------ 2.97 MiB/5.31 MiB
⠹ Preparing packages... (32/33)
pyright ------------------------------ 3.09 MiB/5.31 MiB
⠹ Preparing packages... (32/33)
pyright ------------------------------ 3.23 MiB/5.31 MiB
⠹ Preparing packages... (32/33)
pyright ------------------------------ 3.34 MiB/5.31 MiB
⠸ Preparing packages... (32/33)
pyright ------------------------------ 3.50 MiB/5.31 MiB
⠸ Preparing packages... (32/33)
pyright ------------------------------ 3.58 MiB/5.31 MiB
⠸ Preparing packages... (32/33)
pyright ------------------------------ 3.81 MiB/5.31 MiB
⠸ Preparing packages... (32/33)
pyright ------------------------------ 4.17 MiB/5.31 MiB
⠼ Preparing packages... (32/33)
pyright ------------------------------ 4.39 MiB/5.31 MiB
⠼ Preparing packages... (32/33)
pyright ------------------------------ 4.53 MiB/5.31 MiB
⠼ Preparing packages... (32/33)
Prepared 33 packages in 2.53s
░░░░░░░░░░░░░░░░░░░░ [0/0] Installing wheels...
░░░░░░░░░░░░░░░░░░░░ [0/33] Installing wheels...
░░░░░░░░░░░░░░░░░░░░ [0/33] pluggy==1.6.0
░░░░░░░░░░░░░░░░░░░░ [1/33] pluggy==1.6.0
░░░░░░░░░░░░░░░░░░░░ [1/33] pytest-playwright==0.7.0
█░░░░░░░░░░░░░░░░░░░ [2/33] pytest-playwright==0.7.0
█░░░░░░░░░░░░░░░░░░░ [2/33] text-unidecode==1.3
█░░░░░░░░░░░░░░░░░░░ [3/33] text-unidecode==1.3
█░░░░░░░░░░░░░░░░░░░ [3/33] flask==3.1.1
██░░░░░░░░░░░░░░░░░░ [4/33] flask==3.1.1
██░░░░░░░░░░░░░░░░░░ [4/33] iniconfig==2.1.0
███░░░░░░░░░░░░░░░░░ [5/33] iniconfig==2.1.0
███░░░░░░░░░░░░░░░░░ [5/33] markupsafe==3.0.2
███░░░░░░░░░░░░░░░░░ [6/33] markupsafe==3.0.2
███░░░░░░░░░░░░░░░░░ [6/33] packaging==25.0
████░░░░░░░░░░░░░░░░ [7/33] jinja2==3.1.6
████░░░░░░░░░░░░░░░░ [8/33] jinja2==3.1.6
████░░░░░░░░░░░░░░░░ [8/33] jinja2==3.1.6
████░░░░░░░░░░░░░░░░ [8/33] execnet==2.1.1
█████░░░░░░░░░░░░░░░ [9/33] execnet==2.1.1
███████████████████░ [32/33] pyright==1.1.400
████████████████████ [33/33] pyright==1.1.400
Installed 33 packages in 107ms
+ blinker==1.9.0
+ certifi==2025.4.26
+ charset-normalizer==3.4.2
+ click==8.2.0
+ duckdb==1.2.2
+ execnet==2.1.1
+ flask==3.1.1
+ greenlet==3.2.2
+ idna==3.10
+ iniconfig==2.1.0
+ itsdangerous==2.2.0
+ jinja2==3.1.6
+ markupsafe==3.0.2
+ nodeenv==1.9.1
+ packaging==25.0
+ playwright==1.52.0
+ pluggy==1.6.0
+ pyee==13.0.0
+ pyright==1.1.400
+ pytest==8.3.5
+ pytest-base-url==2.1.0
+ pytest-playwright==0.7.0
+ pytest-xdist==3.6.1
+ python-dateutil==2.9.0.post0
+ python-slugify==8.0.4
+ requests==2.32.3
+ ruff==0.11.10
+ scubaduck==0.1.0 (from file:///workspace/scubaduck)
+ six==1.17.0
+ text-unidecode==1.3
+ typing-extensions==4.13.2
+ urllib3==2.4.0
+ werkzeug==3.1.3
++ source .venv/bin/activate
+++ '[' -z '' ']'
+++ '[' -n x ']'
+++ SCRIPT_PATH=.venv/bin/activate
+++ '[' .venv/bin/activate = /tmp/flbP2D-setup_script.sh ']'
+++ deactivate nondestructive
+++ unset -f pydoc
+++ '[' -z '' ']'
+++ '[' -z '' ']'
+++ hash -r
+++ '[' -z '' ']'
+++ unset VIRTUAL_ENV
+++ unset VIRTUAL_ENV_PROMPT
+++ '[' '!' nondestructive = nondestructive ']'
+++ VIRTUAL_ENV=/workspace/scubaduck/.venv
+++ '[' linux-gnu = cygwin ']'
+++ '[' linux-gnu = msys ']'
+++ export VIRTUAL_ENV
+++ '[' -z '' ']'
+++ unset SCRIPT_PATH
+++ _OLD_VIRTUAL_PATH=/root/.cargo/bin:/root/.rbenv/shims:/root/.rbenv/bin:/root/.rbenv/shims:/root/.local/share/swiftly/bin:/root/.bun/bin:/root/.nvm/versions/node/v22.15.1/bin:/root/.pyenv/shims:3441PYENV_ROOT/shims:/root/.pyenv/bin:/usr/local/go/bin:/root/go/bin:/root/.rbenv/bin:/root/.rbenv/shims:/root/.bun/bin:/root/.local/bin:/root/.pyenv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+++ PATH=/workspace/scubaduck/.venv/bin:/root/.cargo/bin:/root/.rbenv/shims:/root/.rbenv/bin:/root/.rbenv/shims:/root/.local/share/swiftly/bin:/root/.bun/bin:/root/.nvm/versions/node/v22.15.1/bin:/root/.pyenv/shims:3441PYENV_ROOT/shims:/root/.pyenv/bin:/usr/local/go/bin:/root/go/bin:/root/.rbenv/bin:/root/.rbenv/shims:/root/.bun/bin:/root/.local/bin:/root/.pyenv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+++ export PATH
+++ '[' xscubaduck '!=' x ']'
+++ VIRTUAL_ENV_PROMPT='(scubaduck) '
+++ export VIRTUAL_ENV_PROMPT
+++ '[' -z '' ']'
+++ '[' -z '' ']'
+++ _OLD_VIRTUAL_PS1=
+++ PS1='(scubaduck) '
+++ export PS1
+++ alias pydoc
+++ true
+++ hash -r
++ python -c 'import os; import duckdb; con = duckdb.connect(); con.execute(f"SET http_proxy = '\''{os.getenv("HTTP_PROXY")}'\''"); con.execute("INSTALL '\''sqlite'\'';")'
++ playwright install chromium
Downloading Chromium 136.0.7103.25 (playwright build v1169) from https://cdn.playwright.dev/dbazure/download/playwright/builds/chromium/1169/chromium-linux.zip
167.7 MiB [] 0% 0.0s167.7 MiB [] 0% 43.6s167.7 MiB [] 0% 48.1s167.7 MiB [] 0% 34.7s167.7 MiB [] 0% 25.9s167.7 MiB [] 0% 16.7s167.7 MiB [] 1% 10.6s167.7 MiB [] 1% 8.6s167.7 MiB [] 2% 5.9s167.7 MiB [] 2% 5.3s167.7 MiB [] 4% 4.0s167.7 MiB [] 5% 3.4s167.7 MiB [] 6% 3.1s167.7 MiB [] 7% 2.9s167.7 MiB [] 8% 2.7s167.7 MiB [] 9% 2.5s167.7 MiB [] 10% 2.4s167.7 MiB [] 11% 2.2s167.7 MiB [] 12% 2.1s167.7 MiB [] 13% 2.0s167.7 MiB [] 15% 1.9s167.7 MiB [] 15% 2.0s167.7 MiB [] 16% 2.0s167.7 MiB [] 17% 1.9s167.7 MiB [] 18% 1.8s167.7 MiB [] 19% 1.8s167.7 MiB [] 20% 1.7s167.7 MiB [] 22% 1.7s167.7 MiB [] 22% 1.6s167.7 MiB [] 23% 1.6s167.7 MiB [] 24% 1.6s167.7 MiB [] 25% 1.5s167.7 MiB [] 26% 1.5s167.7 MiB [] 27% 1.5s167.7 MiB [] 29% 1.4s167.7 MiB [] 30% 1.4s167.7 MiB [] 32% 1.3s167.7 MiB [] 33% 1.3s167.7 MiB [] 34% 1.3s167.7 MiB [] 35% 1.2s167.7 MiB [] 37% 1.2s167.7 MiB [] 38% 1.1s167.7 MiB [] 39% 1.1s167.7 MiB [] 40% 1.1s167.7 MiB [] 42% 1.0s167.7 MiB [] 43% 1.0s167.7 MiB [] 44% 1.0s167.7 MiB [] 45% 1.0s167.7 MiB [] 46% 1.0s167.7 MiB [] 47% 0.9s167.7 MiB [] 48% 0.9s167.7 MiB [] 50% 0.9s167.7 MiB [] 52% 0.8s167.7 MiB [] 53% 0.8s167.7 MiB [] 55% 0.7s167.7 MiB [] 56% 0.7s167.7 MiB [] 57% 0.7s167.7 MiB [] 58% 0.7s167.7 MiB [] 59% 0.7s167.7 MiB [] 60% 0.7s167.7 MiB [] 61% 0.6s167.7 MiB [] 63% 0.6s167.7 MiB [] 64% 0.6s167.7 MiB [] 66% 0.5s167.7 MiB [] 67% 0.5s167.7 MiB [] 69% 0.5s167.7 MiB [] 71% 0.5s167.7 MiB [] 72% 0.4s167.7 MiB [] 73% 0.4s167.7 MiB [] 74% 0.4s167.7 MiB [] 76% 0.4s167.7 MiB [] 77% 0.4s167.7 MiB [] 78% 0.3s167.7 MiB [] 79% 0.3s167.7 MiB [] 80% 0.3s167.7 MiB [] 81% 0.3s167.7 MiB [] 82% 0.3s167.7 MiB [] 83% 0.3s167.7 MiB [] 84% 0.2s167.7 MiB [] 85% 0.2s167.7 MiB [] 86% 0.2s167.7 MiB [] 87% 0.2s167.7 MiB [] 88% 0.2s167.7 MiB [] 89% 0.2s167.7 MiB [] 90% 0.1s167.7 MiB [] 91% 0.1s167.7 MiB [] 92% 0.1s167.7 MiB [] 93% 0.1s167.7 MiB [] 94% 0.1s167.7 MiB [] 95% 0.1s167.7 MiB [] 96% 0.1s167.7 MiB [] 97% 0.0s167.7 MiB [] 98% 0.0s167.7 MiB [] 99% 0.0s167.7 MiB [] 100% 0.0s
Chromium 136.0.7103.25 (playwright build v1169) downloaded to /root/.cache/ms-playwright/chromium-1169
Downloading FFMPEG playwright build v1011 from https://cdn.playwright.dev/dbazure/download/playwright/builds/ffmpeg/1011/ffmpeg-linux.zip
2.3 MiB [] 0% 0.0s2.3 MiB [] 4% 0.4s2.3 MiB [] 10% 0.3s2.3 MiB [] 28% 0.1s2.3 MiB [] 46% 0.1s2.3 MiB [] 85% 0.0s2.3 MiB [] 100% 0.0s
FFMPEG playwright build v1011 downloaded to /root/.cache/ms-playwright/ffmpeg-1011
Downloading Chromium Headless Shell 136.0.7103.25 (playwright build v1169) from https://cdn.playwright.dev/dbazure/download/playwright/builds/chromium/1169/chromium-headless-shell-linux.zip
101.4 MiB [] 0% 0.0s101.4 MiB [] 0% 21.6s101.4 MiB [] 0% 30.5s101.4 MiB [] 0% 13.6s101.4 MiB [] 0% 8.8s101.4 MiB [] 1% 7.0s101.4 MiB [] 2% 3.6s101.4 MiB [] 4% 2.6s101.4 MiB [] 5% 2.4s101.4 MiB [] 6% 2.0s101.4 MiB [] 8% 1.7s101.4 MiB [] 9% 1.6s101.4 MiB [] 11% 1.5s101.4 MiB [] 13% 1.4s101.4 MiB [] 15% 1.2s101.4 MiB [] 17% 1.1s101.4 MiB [] 20% 1.0s101.4 MiB [] 22% 1.0s101.4 MiB [] 24% 0.9s101.4 MiB [] 26% 0.9s101.4 MiB [] 29% 0.8s101.4 MiB [] 32% 0.7s101.4 MiB [] 34% 0.7s101.4 MiB [] 36% 0.7s101.4 MiB [] 37% 0.6s101.4 MiB [] 40% 0.6s101.4 MiB [] 43% 0.6s101.4 MiB [] 45% 0.5s101.4 MiB [] 47% 0.5s101.4 MiB [] 50% 0.5s101.4 MiB [] 53% 0.4s101.4 MiB [] 56% 0.4s101.4 MiB [] 60% 0.3s101.4 MiB [] 62% 0.3s101.4 MiB [] 65% 0.3s101.4 MiB [] 67% 0.3s101.4 MiB [] 70% 0.2s101.4 MiB [] 73% 0.2s101.4 MiB [] 76% 0.2s101.4 MiB [] 79% 0.2s101.4 MiB [] 82% 0.1s101.4 MiB [] 85% 0.1s101.4 MiB [] 87% 0.1s101.4 MiB [] 89% 0.1s101.4 MiB [] 91% 0.1s101.4 MiB [] 93% 0.1s101.4 MiB [] 95% 0.0s101.4 MiB [] 98% 0.0s101.4 MiB [] 100% 0.0s
Chromium Headless Shell 136.0.7103.25 (playwright build v1169) downloaded to /root/.cache/ms-playwright/chromium_headless_shell-1169
++ echo 'source .venv/bin/activate'I’ll check the AGENTS.md for instructions, focusing on handling time column functionality.
ls
bash: .venv/bin/activate: No such file or directory
root@ee1d42b3cdab:/workspace# ls
scubaduckroot@ee1d42b3cdab:/workspace# cd scubaduck
root@ee1d42b3cdab:/workspace/scubaduck# ls
AGENTS.md codex_setup.sh scubaduck tests
README.md pyproject.toml stubs uv.lockroot@ee1d42b3cdab:/workspace/scubaduck# cat AGENTS.md
- Use "source .venv/bin/activate" to activate the virtual environment before
running any commands
- ONLY use non-interactive commands like cat, sed, apply_patch to do edits.
Do NOT use interactive editors.
- Do NOT attempt to install packages. Only the packages specified in
pyproject.toml are available. You cannot add new packages. If you
desperately want another package, make a note of it in the final PR
description.
- Use conventional commits to format PR title
- There are no nested AGENTS.md files, this is the only agents file
- When using Playwright, ONLY use chromium browser. This browser is already
installed.
- Use "ruff check" to check lint, "ruff format" to autoformat files and
"pyright" to typecheck.
- When writing the PR description, include the original prompt supplied by the
user.
- When you add functionality to the server, add server tests to
tests/test_server.py. When you add features to the frontend, add frontend
tests to tests/test_web.pyAGENTS provides guidelines: activate the virtual environment, use specific tools, avoid package installations, and more.
root@ee1d42b3cdab:/workspace/scubaduck# ls -R
.:
AGENTS.md codex_setup.sh scubaduck tests
README.md pyproject.toml stubs uv.lock
./scubaduck:
__init__.py sample.csv server.py static
./scubaduck/static:
index.html js
./scubaduck/static/js:
chip_input.js timeseries_chart.js
./stubs:
duckdb
./stubs/duckdb:
__init__.pyi
./tests:
conftest.py test_server.py test_web.pyTime to look over the server script and the static JS files.
root@ee1d42b3cdab:/workspace/scubaduck# sed -n '1,160p' scubaduck/server.py
from __future__ import annotations
from dataclasses import dataclass, field, replace
from typing import Any, Dict, List, Tuple, cast
import re
from datetime import datetime, timedelta, timezone
import time
from pathlib import Path
import os
import traceback
import math
import duckdb
from dateutil import parser as dtparser
from flask import Flask, jsonify, request, send_from_directory
@dataclass
class Filter:
column: str
op: str
value: str | int | float | list[str] | None
@dataclass
class QueryParams:
start: str | None = None
end: str | None = None
order_by: str | None = None
order_dir: str = "ASC"
limit: int | None = None
columns: list[str] = field(default_factory=lambda: [])
filters: list[Filter] = field(default_factory=lambda: [])
derived_columns: dict[str, str] = field(default_factory=lambda: {})
graph_type: str = "samples"
group_by: list[str] = field(default_factory=lambda: [])
aggregate: str | None = None
show_hits: bool = False
x_axis: str | None = None
granularity: str = "Auto"
fill: str = "0"
table: str = "events"
time_column: str = "timestamp"
time_unit: str = "s"
def _load_database(path: Path) -> duckdb.DuckDBPyConnection:
if not path.exists():
raise FileNotFoundError(path)
ext = path.suffix.lower()
if ext == ".csv":
con = duckdb.connect()
con.execute(
f"CREATE TABLE events AS SELECT * FROM read_csv_auto('{path.as_posix
()}')"
)
elif ext in {".db", ".sqlite"}:
con = duckdb.connect()
con.execute("LOAD sqlite")
con.execute(f"ATTACH '{path.as_posix()}' AS db (TYPE SQLITE)")
tables = [
r[0]
for r in con.execute(
"SELECT name FROM sqlite_master WHERE type='table'"
).fetchall()
]
for t in tables:
con.execute(f'CREATE VIEW "{t}" AS SELECT * FROM db."{t}"')
else:
con = duckdb.connect(path)
return con
_REL_RE = re.compile(
r"([+-]?\d+(?:\.\d*)?)\s*(hour|hours|day|days|week|weeks|fortnight|fortnight
s)",
re.IGNORECASE,
)
def parse_time(val: str | None) -> str | None:
"""Parse an absolute or relative time string into ``YYYY-MM-DD HH:MM:SS``.""
"
if val is None or val == "":
return None
s = val.strip()
if s.lower() == "now":
dt = datetime.now(timezone.utc)
return dt.replace(microsecond=0).strftime("%Y-%m-%d %H:%M:%S")
m = _REL_RE.fullmatch(s)
if m:
qty = float(m.group(1))
unit = m.group(2).lower()
delta: timedelta
if unit.startswith("hour"):
delta = timedelta(hours=qty)
elif unit.startswith("day"):
delta = timedelta(days=qty)
elif unit.startswith("week"):
delta = timedelta(weeks=qty)
elif unit.startswith("fortnight"):
delta = timedelta(weeks=2 * qty)
else: # pragma: no cover - defensive
raise ValueError(f"Unsupported unit: {unit}")
dt = datetime.now(timezone.utc) + delta
return dt.replace(microsecond=0).strftime("%Y-%m-%d %H:%M:%S")
dt = dtparser.parse(s)
return dt.replace(microsecond=0, tzinfo=None).strftime("%Y-%m-%d %H:%M:%S")
def _granularity_seconds(granularity: str, start: str | None, end: str | None) -
> int:
gran = granularity.lower()
mapping = {
"1 second": 1,
"5 seconds": 5,
"10 seconds": 10,
"30 seconds": 30,
"1 minute": 60,
"4 minutes": 240,
"5 minutes": 300,
"10 minutes": 600,
"15 minutes": 900,
"30 minutes": 1800,
"1 hour": 3600,
"3 hours": 10800,
"6 hours": 21600,
"1 day": 86400,
"1 week": 604800,
"30 days": 2592000,
}
if gran in mapping:
return mapping[gran]
if gran in {"auto", "fine"} and start and end:
try:
s = dtparser.parse(start)
e = dtparser.parse(end)
except Exception:
return 3600
total = max((e - s).total_seconds(), 1)
buckets = 100 if gran == "auto" else 500
return max(int(total // buckets), 1)
return 3600
def _time_expr(col: str, column_types: Dict[str, str] | None, unit: str) -> str:
"""Return SQL expression for column interpreted as timestamp."""
if column_types is None:
return col
ctype = column_types.get(col, "").upper()
if not any(t in ctype for t in ["TIMESTAMP", "DATE", "TIME"]):
if any(
t in ctype
for t in [
"INT",
"DECIMAL",
"REAL",
"DOUBLE",
"FLOAT",root@ee1d42b3cdab:/workspace/scubaduck# sed -n '160,320p' scubaduck/server.py
"FLOAT",
"NUMERIC",
"HUGEINT",
]
):
if unit == "ns":
expr = f"CAST({col} AS BIGINT)"
return f"make_timestamp_ns({expr})"
multiplier = {
"s": 1_000_000,
"ms": 1_000,
"us": 1,
}.get(unit, 1_000_000)
expr = (
f"CAST({col} * {multiplier} AS BIGINT)"
if multiplier != 1
else f"CAST({col} AS BIGINT)"
)
return f"make_timestamp({expr})"
return col
def build_query(params: QueryParams, column_types: Dict[str, str] | None = None)
-> str:
select_parts: list[str] = []
group_cols = params.group_by[:]
if params.graph_type == "timeseries":
sec = _granularity_seconds(params.granularity, params.start, params.end)
x_axis = params.x_axis or params.time_column
xexpr = _time_expr(x_axis, column_types, params.time_unit)
if params.start:
bucket_expr = (
f"TIMESTAMP '{params.start}' + INTERVAL '{sec} second' * "
f"CAST(floor((epoch({xexpr}) - epoch(TIMESTAMP '{params.start}')
)/{sec}) AS BIGINT)"
)
else:
bucket_expr = (
f"TIMESTAMP 'epoch' + INTERVAL '{sec} second' * "
f"CAST(floor(epoch({xexpr})/{sec}) AS BIGINT)"
)
select_parts.append(f"{bucket_expr} AS bucket")
group_cols = ["bucket"] + group_cols
has_agg = bool(group_cols) or params.aggregate is not None
if has_agg:
select_cols = (
group_cols[1:] if params.graph_type == "timeseries" else group_cols
)
select_parts.extend(select_cols)
agg = (params.aggregate or "avg").lower()
def agg_expr(col: str) -> str:
expr = col
ctype = column_types.get(col, "").upper() if column_types else ""
if "BOOL" in ctype:
expr = f"CAST({col} AS BIGINT)"
if agg.startswith("p"):
quant = float(agg[1:]) / 100
return f"quantile({expr}, {quant})"
if agg == "count distinct":
return f"count(DISTINCT {expr})"
if agg == "avg" and column_types is not None:
if "TIMESTAMP" in ctype or "DATE" in ctype or "TIME" in ctype:
return (
"TIMESTAMP 'epoch' + INTERVAL '1 second' * "
f"CAST(avg(epoch({col})) AS BIGINT)"
)
return f"{agg}({expr})"
for col in params.columns:
if col in group_cols:
continue
select_parts.append(f"{agg_expr(col)} AS {col}")
if params.show_hits:
select_parts.insert(len(group_cols), "count(*) AS Hits")
else:
select_parts.extend(params.columns)
if has_agg and params.derived_columns:
inner_params = replace(
params,
derived_columns={},
order_by=None,
limit=None,
)
inner_sql = build_query(inner_params, column_types)
outer_select = ["t.*"] + [
f"{expr} AS {name}" for name, expr in params.derived_columns.items()
]
indented_inner = "\n".join(" " + line for line in inner_sql.splitline
s())
lines = [
f"SELECT {', '.join(outer_select)}",
"FROM (",
indented_inner,
") t",
]
if params.order_by:
lines.append(f"ORDER BY {params.order_by} {params.order_dir}")
elif params.graph_type == "timeseries":
lines.append("ORDER BY bucket")
if params.limit is not None:
lines.append(f"LIMIT {params.limit}")
return "\n".join(lines)
for name, expr in params.derived_columns.items():
select_parts.append(f"{expr} AS {name}")
select_clause = ", ".join(select_parts) if select_parts else "*"
lines = [f"SELECT {select_clause}", f'FROM "{params.table}"']
where_parts: list[str] = []
time_expr = _time_expr(params.time_column, column_types, params.time_unit)
if params.start:
where_parts.append(f"{time_expr} >= '{params.start}'")
if params.end:
where_parts.append(f"{time_expr} <= '{params.end}'")
for f in params.filters:
op = f.op
if op in {"empty", "!empty"}:
val = "''"
else:
if f.value is None:
continue
if isinstance(f.value, list):
if not f.value:
continue
if op == "=":
vals = " OR ".join(f"{f.column} = '{v}'" for v in f.value)
where_parts.append(f"({vals})")
continue
val = f"'{f.value}'" if isinstance(f.value, str) else str(f.value)
if op == "contains":
where_parts.append(f"{f.column} ILIKE '%' || {val} || '%'")
elif op == "!contains":
where_parts.append(f"{f.column} NOT ILIKE '%' || {val} || '%'")
elif op == "empty":
where_parts.append(f"{f.column} = {val}")
elif op == "!empty":
where_parts.append(f"{f.column} != {val}")
else:
where_parts.append(f"{f.column} {op} {val}")
if where_parts:
lines.append("WHERE " + " AND ".join(where_parts))
if group_cols:
lines.append("GROUP BY " + ", ".join(group_cols))
if params.order_by:
lines.append(f"ORDER BY {params.order_by} {params.order_dir}")
elif params.graph_type == "timeseries":
lines.append("ORDER BY bucket")
if params.limit is not None:
lines.append(f"LIMIT {params.limit}")
return "\n".join(lines)
def create_app(db_file: str | Path | None = None) -> Flask:
app = Flask(__name__, static_folder="static")
if db_file is None:
env_db = os.environ.get("SCUBADUCK_DB")
if env_db:
db_file = env_db
db_path = Path(db_file or Path(__file__).with_name("sample.csv")).resolve()
con = _load_database(db_path)
tables = [r[0] for r in con.execute("SHOW TABLES").fetchall()]root@ee1d42b3cdab:/workspace/scubaduck# sed -n '320,640p' scubaduck/server.py
tables = [r[0] for r in con.execute("SHOW TABLES").fetchall()]
if not tables:
raise ValueError("No tables found in database")
default_table = tables[0]
columns_cache: Dict[str, Dict[str, str]] = {}
def get_columns(table: str) -> Dict[str, str]:
if table not in columns_cache:
rows = con.execute(f'PRAGMA table_info("{table}")').fetchall()
if not rows:
raise ValueError(f"Unknown table: {table}")
columns_cache[table] = {r[1]: r[2] for r in rows}
return columns_cache[table]
sample_cache: Dict[Tuple[str, str, str], Tuple[List[str], float]] = {}
CACHE_TTL = 60.0
CACHE_LIMIT = 200
@app.route("/")
def index() -> Any: # pyright: ignore[reportUnusedFunction]
assert app.static_folder is not None
return send_from_directory(app.static_folder, "index.html")
@app.route("/js/<path:filename>")
def js(filename: str) -> Any: # pyright: ignore[reportUnusedFunction]
assert app.static_folder is not None
folder = Path(app.static_folder) / "js"
return send_from_directory(folder, filename)
@app.route("/api/tables")
def tables_endpoint() -> Any: # pyright: ignore[reportUnusedFunction]
return jsonify(tables)
@app.route("/api/columns")
def columns() -> Any: # pyright: ignore[reportUnusedFunction]
table = request.args.get("table", default_table)
rows = con.execute(f'PRAGMA table_info("{table}")').fetchall()
return jsonify([{"name": r[1], "type": r[2]} for r in rows])
def _cache_get(key: Tuple[str, str, str]) -> List[str] | None:
item = sample_cache.get(key)
if item is None:
return None
vals, ts = item
if time.time() - ts > CACHE_TTL:
del sample_cache[key]
return None
sample_cache[key] = (vals, time.time())
return vals
def _cache_set(key: Tuple[str, str, str], vals: List[str]) -> None:
sample_cache[key] = (vals, time.time())
if len(sample_cache) > CACHE_LIMIT:
oldest = min(sample_cache.items(), key=lambda kv: kv[1][1])[0]
del sample_cache[oldest]
@app.route("/api/samples")
def sample_values() -> Any: # pyright: ignore[reportUnusedFunction]
table = request.args.get("table", default_table)
column = request.args.get("column")
substr = request.args.get("q", "")
column_types = get_columns(table)
if not column or column not in column_types:
return jsonify([])
ctype = column_types[column].upper()
if "CHAR" not in ctype and "STRING" not in ctype and "VARCHAR" not in ct
ype:
return jsonify([])
key = (table, column, substr)
cached = _cache_get(key)
if cached is not None:
return jsonify(cached)
rows = con.execute(
f"SELECT DISTINCT {column} FROM \"{table}\" WHERE CAST({column} AS V
ARCHAR) ILIKE '%' || ? || '%' LIMIT 20",
[substr],
).fetchall()
values = [r[0] for r in rows]
_cache_set(key, values)
return jsonify(values)
@app.route("/api/query", methods=["POST"])
def query() -> Any: # pyright: ignore[reportUnusedFunction]
payload = request.get_json(force=True)
try:
start = parse_time(payload.get("start"))
end = parse_time(payload.get("end"))
except Exception as exc:
return jsonify({"error": str(exc)}), 400
params = QueryParams(
start=start,
end=end,
order_by=payload.get("order_by"),
order_dir=payload.get("order_dir", "ASC"),
limit=payload.get("limit"),
columns=payload.get("columns", []),
derived_columns=payload.get("derived_columns", {}),
graph_type=payload.get("graph_type", "samples"),
group_by=payload.get("group_by", []),
aggregate=payload.get("aggregate"),
show_hits=payload.get("show_hits", False),
x_axis=payload.get("x_axis"),
granularity=payload.get("granularity", "Auto"),
fill=payload.get("fill", "0"),
table=payload.get("table", default_table),
time_column=payload.get("time_column", "timestamp"),
time_unit=payload.get("time_unit", "s"),
)
for f in payload.get("filters", []):
params.filters.append(Filter(f["column"], f["op"], f.get("value")))
if params.table not in tables:
return jsonify({"error": "Invalid table"}), 400
column_types = get_columns(params.table)
if params.time_column not in column_types:
return jsonify({"error": "Invalid time_column"}), 400
if params.time_unit not in {"s", "ms", "us", "ns"}:
return jsonify({"error": "Invalid time_unit"}), 400
...
to column {c}",
}
),
400,
)
if allow_time and not (is_numeric or is_time):
return (
jsonify(
{
"error": f"Aggregate {agg} cannot be applied
to column {c}",
}
),
400,
)
if params.start is None or params.end is None:
axis = params.x_axis or params.time_column
row = cast(
tuple[datetime | None, datetime | None],
con.execute(
f'SELECT min({axis}), max({axis}) FROM "{params.table}"'
).fetchall()[0],
)
mn, mx = row
divisor = {
"s": 1,
"ms": 1000,
"us": 1_000_000,
"ns": 1_000_000_000,
}.get(params.time_unit, 1)
if isinstance(mn, (int, float)):
try:
mn = datetime.fromtimestamp(int(mn) / divisor, tz=timezone.u
tc)
except Exception:
return (
jsonify(
{
"error": (
f"Invalid time value {mn} for column {axis}"
f" with time_unit {params.time_unit}"
)
}
),
400,
)
if isinstance(mx, (int, float)):
try:
mx = datetime.fromtimestamp(int(mx) / divisor, tz=timezone.u
tc)
except Exception:
return (
jsonify(
{
"error": (
f"Invalid time value {mx} for column {axis}"
f" with time_unit {params.time_unit}"
)
}
),
400,
)
if params.start is None and mn is not None:
params.start = (
mn.strftime("%Y-%m-%d %H:%M:%S") if not isinstance(mn, str)
else mn
)
if params.end is None and mx is not None:
params.end = (
mx.strftime("%Y-%m-%d %H:%M:%S") if not isinstance(mx, str)
else mx
)
bucket_size: int | None = None
if params.graph_type == "timeseries":
bucket_size = _granularity_seconds(
params.granularity,
params.start if isinstance(params.start, str) else None,
params.end if isinstance(params.end, str) else None,
)
if (
params.limit is not None
and params.start is not None
and params.end is not None
):
try:
start_dt = dtparser.parse(params.start)
end_dt = dtparser.parse(params.end)
buckets = math.ceil(
(end_dt - start_dt).total_seconds() / bucket_size
)
if buckets > 1:
params.limit *= buckets
except Exception:
pass
sql = build_query(params, column_types)
try:
rows = con.execute(sql).fetchall()
except Exception as exc:
tb = traceback.format_exc()
print(f"Query failed:\n{sql}\n{tb}")
return (
jsonify({"sql": sql, "error": str(exc), "traceback": tb}),
400,
)
result: Dict[str, Any] = {"sql": sql, "rows": rows}
if params.start is not None:
result["start"] = str(params.start)
if params.end is not None:
result["end"] = str(params.end)
if bucket_size is not None:
result["bucket_size"] = bucket_size
return jsonify(result)
return app
app = create_app()
if __name__ == "__main__":
app.run(debug=True)I need to adjust front-end for letting users set time_column to (none), hiding start/end fields.
root@ee1d42b3cdab:/workspace/scubaduck# find scubaduck/static/js -maxdepth 1 -ty
pe f -print
scubaduck/static/js/timeseries_chart.js
scubaduck/static/js/chip_input.jsroot@ee1d42b3cdab:/workspace/scubaduck# sed -n '1,160p' scubaduck/static/js/time
series_chart.js
let resizeObserver = null;
let currentChart = null;
function showTimeSeries(data) {
function parseTs(s) {
if (s.match(/GMT/) || s.endsWith('Z') || /\+\d{2}:?\d{2}$/.test(s)) {
return new Date(s).getTime();
}
return new Date(s + 'Z').getTime();
}
const view = document.getElementById('view');
if (data.rows.length === 0) {
view.innerHTML = '<p id="empty-message">Empty data provided to table</p>';
return;
}
const height = 600;
view.innerHTML =
'<div id="ts-container"><div id="legend"></div><div id="chart-wrapper"><svg
id="chart" height="' +
height +
'"></svg></div></div>';
const svg = document.getElementById('chart');
const legend = document.getElementById('legend');
const crosshairLine = document.createElementNS('http://www.w3.org/2000/svg', '
line');
crosshairLine.id = 'crosshair_line';
crosshairLine.setAttribute('stroke', '#555');
crosshairLine.style.display = 'none';
const crosshairDots = document.createElementNS('http://www.w3.org/2000/svg', '
g');
crosshairDots.id = 'crosshair_dots';
crosshairDots.style.display = 'none';
const groups = groupBy.chips || [];
const hasHits = document.getElementById('show_hits').checked ? 1 : 0;
const fill = document.getElementById('fill').value;
const bucketMs = (data.bucket_size || 3600) * 1000;
const start = data.start ? parseTs(data.start) : null;
const end = data.end ? parseTs(data.end) : null;
const startIdx = 1 + groups.length + hasHits;
const valueCols = selectedColumns.slice(groups.length + hasHits);
const series = {};
data.rows.forEach(r => {
const ts = parseTs(r[0]);
const groupKey = groups.map((_, i) => r[1 + i]).join(':') || 'all';
valueCols.forEach((name, i) => {
const val = Number(r[startIdx + i]);
const key = groupKey === 'all' ? name : groupKey + ':' + name;
if (!series[key]) series[key] = {};
series[key][ts] = val;
});
});
const buckets = [];
let minX = start !== null ? start : Infinity;
let maxX = end !== null ? end : -Infinity;
if (start !== null && end !== null) {
for (let t = start; t <= end; t += bucketMs) {
buckets.push(t);
}
} else {
Object.keys(series).forEach(k => {
const s = series[k];
Object.keys(s).forEach(t => {
const n = Number(t);
if (n < minX) minX = n;
if (n > maxX) maxX = n;
});
});
for (let t = minX; t <= maxX; t += bucketMs) {
buckets.push(t);
}
}
let minY = Infinity,
maxY = -Infinity;
Object.keys(series).forEach(key => {
const vals = series[key];
buckets.forEach(b => {
const v = vals[b];
const val = v === undefined && fill === '0' ? 0 : v;
if (val === undefined) return;
if (val < minY) minY = val;
if (val > maxY) maxY = val;
});
});
if (fill === '0') {
if (minY > 0) minY = 0;
if (maxY < 0) maxY = 0;
}
const colors = [
'#1f77b4',
'#ff7f0e',
'#2ca02c',
'#d62728',
'#9467bd',
'#8c564b',
'#e377c2'
];
currentChart = {
svg,
legend,
series,
buckets,
minX,
maxX,
minY,
maxY,
fill,
colors,
height,
crosshairLine,
crosshairDots,
seriesEls: {},
bucketPixels: [],
xScale: null,
yScale: null,
selected: null,
frozen: false
};
const intervals = [
{unit: 'second', step: 1, ms: 1000},
{unit: 'second', step: 2, ms: 2000},
{unit: 'second', step: 5, ms: 5000},
{unit: 'second', step: 10, ms: 10000},
{unit: 'second', step: 15, ms: 15000},
{unit: 'second', step: 30, ms: 30000},
{unit: 'minute', step: 1, ms: 60000},
{unit: 'minute', step: 2, ms: 120000},
{unit: 'minute', step: 5, ms: 300000},
{unit: 'minute', step: 10, ms: 600000},
{unit: 'minute', step: 15, ms: 900000},
{unit: 'minute', step: 30, ms: 1800000},
{unit: 'hour', step: 1, ms: 3600000},
{unit: 'hour', step: 2, ms: 7200000},
{unit: 'hour', step: 3, ms: 10800000},
{unit: 'hour', step: 4, ms: 14400000},
{unit: 'hour', step: 6, ms: 21600000},
{unit: 'hour', step: 12, ms: 43200000},
{unit: 'day', step: 1, ms: 86400000},
{unit: 'day', step: 2, ms: 172800000},
{unit: 'week', step: 1, ms: 604800000},
{unit: 'week', step: 2, ms: 1209600000},
{unit: 'month', step: 1},
{unit: 'month', step: 3},
{unit: 'month', step: 6},
{unit: 'year', step: 1},
{unit: 'year', step: 2},
{unit: 'year', step: 5},
{unit: 'year', step: 10}
];
function chooseInterval(start, end) {
const span = end - start;
function approxMs(i) {
if (i.ms) return i.ms;
if (i.unit === 'month') return i.step * 2629800000;
if (i.unit === 'year') return i.step * 31557600000;
return 1000;
}root@ee1d42b3cdab:/workspace/scubaduck# grep -n "start" -n scubaduck/static/js/*
.js
scubaduck/static/js/timeseries_chart.js:35: const start = data.start ? parseTs(
data.start) : null;
scubaduck/static/js/timeseries_chart.js:37: const startIdx = 1 + groups.length
+ hasHits;
scubaduck/static/js/timeseries_chart.js:44: const val = Number(r[startIdx +
i]);
scubaduck/static/js/timeseries_chart.js:52: let minX = start !== null ? start :
Infinity;
scubaduck/static/js/timeseries_chart.js:54: if (start !== null && end !== null)
{
scubaduck/static/js/timeseries_chart.js:55: for (let t = start; t <= end; t +
= bucketMs) {
scubaduck/static/js/timeseries_chart.js:153: function chooseInterval(start, end
) {
scubaduck/static/js/timeseries_chart.js:154: const span = end - start;
scubaduck/static/js/timeseries_chart.js:174: function generateTicks(start, end,
intv) {
scubaduck/static/js/timeseries_chart.js:177: let d = new Date(start);
scubaduck/static/js/timeseries_chart.js:191: if (t >= start) ticks.push(t
);
scubaduck/static/js/timeseries_chart.js:196: let t = Math.ceil(start / step
) * step;
scubaduck/static/js/timeseries_chart.js:203: if (t === start) t += step;
scubaduck/static/js/timeseries_chart.js:261: const start = Math.floor(min / s
tep) * step;
scubaduck/static/js/timeseries_chart.js:264: for (let v = start; v <= end; v
+= step) ticks.push(v);I'll look for the time_column variable to see where it's defined.
root@ee1d42b3cdab:/workspace/scubaduck# grep -n "time_column" -r . | head./tests/test_web.py:111:def test_time_column_dropdown(page: Any, server_url: str
) -> None:
./tests/test_web.py:113: page.wait_for_selector("#time_column option", state=
"attached")
./tests/test_web.py:114: options = page.locator("#time_column option").all_in
ner_texts()
./tests/test_web.py:117: assert page.input_value("#time_column") == "timestam
p"
./tests/test_server.py:300:def test_integer_time_column(tmp_path: Path) -> None:
./tests/test_server.py:311: "time_column": "created",
./tests/test_server.py:332: "time_column": "created",
./tests/test_server.py:343:def test_timeseries_default_xaxis_uses_time_column(tm
p_path: Path) -> None:
./tests/test_server.py:356: "time_column": "created",
./tests/test_server.py:378: "time_column": "created",I'll examine the tests to better understand the features they're testing.
import json
from pathlib import Path
from typing import cast
import duckdb
from scubaduck import server
import pytest
def test_basic_query() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-02 00:00:00",
"order_by": "timestamp",
"order_dir": "ASC",
"limit": 10,
"columns": ["timestamp", "event", "value", "user"],
"filters": [],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert data
rows = data["rows"]
# We expect first three rows (until 2024-01-02 00:00:00)
assert len(rows) == 3
assert rows[0][1] == "login"
assert rows[1][1] == "logout"
def test_js_served() -> None:
app = server.app
client = app.test_client()
rv = client.get("/js/chip_input.js")
assert rv.status_code == 200
assert b"initChipInput" in rv.data
def test_filter_multi_token() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-02 03:00:00",
"order_by": "timestamp",
"limit": 10,
"columns": ["timestamp", "event", "value", "user"],
"filters": [{"column": "user", "op": "=", "value": ["alice", "charlie"]}
],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert data
rows = data["rows"]
# Should only return rows for alice and charlie
assert len(rows) == 3
assert rows[0][3] == "alice"
assert rows[-1][3] == "charlie"
def test_empty_filter_is_noop() -> None:
app = server.app
client = app.test_client()
base_payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"limit": 100,
"columns": ["timestamp", "event", "value", "user"],
}
no_filter = {**base_payload, "filters": []}
empty_filter = {
**base_payload,
"filters": [{"column": "user", "op": "=", "value": None}],
}
rv1 = client.post(
"/api/query", data=json.dumps(no_filter), content_type="application/json
"
)
rv2 = client.post(
"/api/query", data=json.dumps(empty_filter), content_type="application/j
son"
)
rows1 = rv1.get_json()["rows"]
rows2 = rv2.get_json()["rows"]
assert rows1 == rows2
def test_select_columns() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"order_by": "timestamp",
"limit": 10,
"columns": ["timestamp", "user"],
"filters": [],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert data
rows = data["rows"]
assert len(rows[0]) == 2
assert rows[0][1] == "alice"
def test_string_filter_ops() -> None:
app = server.app
client = app.test_client()
base = {
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"order_by": "timestamp",
"limit": 100,
"columns": ["timestamp", "event", "value", "user"],
}
contains = {
**base,
"filters": [{"column": "user", "op": "contains", "value": "ali"}],
}
rv = client.post(
"/api/query", data=json.dumps(contains), content_type="application/json"
)
rows = rv.get_json()["rows"]
assert all("ali" in r[3] for r in rows)
regex = {
**base,
"filters": [{"column": "user", "op": "~", "value": "^a.*"}],
}
rv = client.post(
"/api/query", data=json.dumps(regex), content_type="application/json"
)
rows = rv.get_json()["rows"]
assert all(r[3].startswith("a") for r in rows)
assert len(rows) == 2
not_empty = {**base, "filters": [{"column": "user", "op": "!empty"}]}
rv = client.post(
"/api/query", data=json.dumps(not_empty), content_type="application/json
"
)
assert len(rv.get_json()["rows"]) == 4
def _make_payload() -> dict[str, object]:
return {
"table": "events",
"start": "2024-01-01 00:00:00","limit": 10,
"columns": ["timestamp", "event", "value", "user"],
"filters": [],
}
def test_database_types(tmp_path: Path) -> None:
csv_file = tmp_path / "events.csv"
csv_file.write_text(Path("scubaduck/sample.csv").read_text())
sqlite_file = tmp_path / "events.sqlite"
import sqlite3
conn = sqlite3.connect(sqlite_file)
conn.execute(
"CREATE TABLE events (timestamp TEXT, event TEXT, value INTEGER, user TE
XT)"
)
with open(csv_file) as f:
next(f)
for line in f:
ts, ev, val, user = line.strip().split(",")
conn.execute(
"INSERT INTO events VALUES (?, ?, ?, ?)", (ts, ev, int(val), use
r)
)
conn.commit()
conn.close() # pyright: ignore[reportUnknownMemberType, reportAttributeAcce
ssIssue]
duckdb_file = tmp_path / "events.duckdb"
con = duckdb.connect(duckdb_file)
con.execute(
f"CREATE TABLE events AS SELECT * FROM read_csv_auto('{csv_file.as_posix
()}')"
)
con.close() # pyright: ignore[reportUnknownMemberType, reportAttributeAcces
sIssue]
for db in (csv_file, sqlite_file, duckdb_file):
app = server.create_app(db)
client = app.test_client()
payload = _make_payload()
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/js
on"
)
rows = rv.get_json()["rows"]
assert len(rows) == 3
def test_sqlite_longvarchar(tmp_path: Path) -> None:
sqlite_file = tmp_path / "events.sqlite"
import sqlite3
conn = sqlite3.connect(sqlite_file)
conn.execute(
"CREATE TABLE events (timestamp TEXT, url LONGVARCHAR, title VARCHAR(10)
)"
)
conn.execute(
"INSERT INTO events VALUES ('2024-01-01 00:00:00', 'https://a.com', 'Hom
e')"
)
conn.commit()
conn.close() # pyright: ignore[reportUnknownMemberType, reportAttributeAcce
ssIssue]
app = server.create_app(sqlite_file)
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-01 01:00:00",
"order_by": "timestamp",
"columns": ["timestamp", "url", "title"],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert data["rows"][0][1] == "https://a.com"
def test_sqlite_bigint(tmp_path: Path) -> None:
sqlite_file = tmp_path / "big.sqlite"
import sqlite3
conn = sqlite3.connect(sqlite_file)
conn.execute("CREATE TABLE events (timestamp TEXT, value INTEGER)")
big_value = 13385262862605259
conn.execute(
"INSERT INTO events VALUES ('2024-01-01 00:00:00', ?)",
(big_value,),
)
conn.commit()
conn.close() # pyright: ignore[reportUnknownMemberType, reportAttributeAcce
ssIssue]
app = server.create_app(sqlite_file)
client = app.test_client()
payload = {
"table": "events",
"order_by": "timestamp",
"columns": ["timestamp", "value"],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert data["rows"][0][1] == big_value
def test_sqlite_boolean_aggregation(tmp_path: Path) -> None:
sqlite_file = tmp_path / "bool.sqlite"
import sqlite3
conn = sqlite3.connect(sqlite_file)
conn.execute("CREATE TABLE events (timestamp TEXT, flag BOOLEAN)")
conn.execute("INSERT INTO events VALUES ('2024-01-01 00:00:00', 1)")
conn.execute("INSERT INTO events VALUES ('2024-01-01 00:30:00', 0)")
conn.commit()
conn.close() # pyright: ignore[reportUnknownMemberType, reportAttributeAcce
ssIssue]
app = server.create_app(sqlite_file)
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-02 00:00:00",
"graph_type": "table",
"aggregate": "Avg",
"columns": ["flag"],
"show_hits": True,
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert data["rows"][0][0] == 2
assert data["rows"][0][1] == 0.5
def test_integer_time_column(tmp_path: Path) -> None:
csv_file = tmp_path / "events.csv"
csv_file.write_text("created,event\n1704067200,login\n1704070800,logout\n")
app = server.create_app(csv_file)
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-01 01:00:00",
"order_by": "created",
"columns": ["created", "event"],
"time_column": "created",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert len(data["rows"]) == 2"time_unit": "ms",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert len(data["rows"]) == 2
def test_timeseries_default_xaxis_uses_time_column(tmp_path: Path) -> None:
csv_file = tmp_path / "events.csv"
csv_file.write_text("created,event\n1704067200000,login\n1704070800000,logou
t\n")
app = server.create_app(csv_file)
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-01 01:00:00",
"graph_type": "timeseries",
"granularity": "1 hour",
"columns": ["event"],
"aggregate": "Count",
"time_column": "created",
"time_unit": "ms",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert len(data["rows"]) == 2
def test_integer_time_unit_us_default_start_end(tmp_path: Path) -> None:
csv_file = tmp_path / "events.csv"
csv_file.write_text(
"created,event\n1704067200000000,login\n1704070800000000,logout\n"
)
app = server.create_app(csv_file)
client = app.test_client()
payload = {
"table": "events",
"order_by": "created",
"columns": ["created", "event"],
"time_column": "created",
"time_unit": "us",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert data["start"] == "2024-01-01 00:00:00"
assert data["end"] == "2024-01-01 01:00:00"
assert len(data["rows"]) == 2
def test_sqlite_integer_time_unit_us(tmp_path: Path) -> None:
sqlite_file = tmp_path / "events.sqlite"
import sqlite3
conn = sqlite3.connect(sqlite_file)
conn.execute("CREATE TABLE visits (visit_time INTEGER, event TEXT)")
big_ts = 13384551652000000
conn.execute("INSERT INTO visits VALUES (?, ?)", (big_ts, "foo"))
conn.commit()
conn.close() # pyright: ignore[reportUnknownMemberType, reportAttributeAcce
ssIssue]
app = server.create_app(sqlite_file)
client = app.test_client()
payload = {
"table": "visits",
"start": "2394-02-20 00:00:00",
"end": "2394-02-21 00:00:00",
"order_by": "visit_time",
"columns": ["visit_time", "event"],
"time_column": "visit_time",
"time_unit": "us",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert len(data["rows"]) == 1
def test_envvar_db(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
csv_file = tmp_path / "custom.csv"
csv_file.write_text("timestamp,event,value,user\n2024-01-01 00:00:00,login,5
,bob\n")
monkeypatch.setenv("SCUBADUCK_DB", str(csv_file))
app = server.create_app()
client = app.test_client()
payload = _make_payload()
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
rows = rv.get_json()["rows"]
assert len(rows) == 1
def test_envvar_db_missing(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> N
one:
missing = tmp_path / "missing.sqlite"
monkeypatch.setenv("SCUBADUCK_DB", str(missing))
with pytest.raises(FileNotFoundError):
server.create_app()
def test_group_by_table() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "table",
"order_by": "user",
"limit": 10,
"columns": ["value"],
...
def test_query_error_returns_sql_and_traceback() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"columns": ["event"],
"group_by": ["user"],
"aggregate": "avg",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 400
assert "error" in data
def test_table_unknown_column_error() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "table",
"order_by": "timestamp",
"limit": 100,
"columns": ["user", "Hits", "value"],
"group_by": ["user"],
"aggregate": "Count",
"show_hits": True,
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 400
assert "Unknown column" in data["error"]
def test_samples_view_rejects_group_by() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"graph_type": "samples",
"group_by": ["user"],
"columns": ["timestamp"],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 400
assert "only valid" in data["error"]
def test_table_avg_with_timestamp() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "table",
"order_by": "user",
"limit": 100,
"columns": ["user", "timestamp", "value"],
"group_by": ["user"],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert "error" not in data
rows = data["rows"]
assert rows[0][0] == "alice"
from dateutil import parser
ts = parser.parse(rows[0][1]).replace(tzinfo=None)
assert ts == parser.parse("2024-01-01 12:00:00")
def test_timeseries_basic() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "timeseries",
"limit": 100,
"group_by": ["user"],
"aggregate": "Count",
"columns": ["value"],
"x_axis": "timestamp",
"granularity": "1 day",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert len(data["rows"]) == 4
def test_timeseries_orders_by_xaxis() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "timeseries",
"limit": 100,
"columns": ["value"],
"x_axis": "timestamp",
"granularity": "1 day",
}root@ee1d42b3cdab:/workspace/scubaduck# grep -n "time_column" -n tests/test_serv
er.py
300:def test_integer_time_column(tmp_path: Path) -> None:
311: "time_column": "created",
332: "time_column": "created",
343:def test_timeseries_default_xaxis_uses_time_column(tmp_path: Path) -> None:
356: "time_column": "created",
378: "time_column": "created",
410: "time_column": "visit_time",I'll focus on the lines roughly between 300 and 332 to review them.
csv_file.write_text("created,event\n1704067200,login\n1704070800,logout\n")
app = server.create_app(csv_file)
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-01 01:00:00",
"order_by": "created",
"columns": ["created", "event"],
"time_column": "created",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert len(data["rows"]) == 2
def test_integer_time_unit_ms(tmp_path: Path) -> None:
csv_file = tmp_path / "events.csv"
csv_file.write_text("created,event\n1704067200000,login\n1704070800000,logou
t\n")
app = server.create_app(csv_file)
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-01 01:00:00",
"order_by": "created",
"columns": ["created", "event"],
"time_column": "created",
"time_unit": "ms",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert len(data["rows"]) == 2
def test_timeseries_default_xaxis_uses_time_column(tmp_path: Path) -> None:
csv_file = tmp_path / "events.csv"
csv_file.write_text("created,event\n1704067200000,login\n1704070800000,logou
t\n")
app = server.create_app(csv_file)
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-01 01:00:00",
"graph_type": "timeseries",
"granularity": "1 hour",
"columns": ["event"],
"aggregate": "Count",
"time_column": "created",
"time_unit": "ms",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert len(data["rows"]) == 2
def test_integer_time_unit_us_default_start_end(tmp_path: Path) -> None:
csv_file = tmp_path / "events.csv"
csv_file.write_text(
"created,event\n1704067200000000,login\n1704070800000000,logout\n"
)
app = server.create_app(csv_file)
client = app.test_client()
payload = {
"table": "events",
"order_by": "created",
"columns": ["created", "event"],
"time_column": "created",
"time_unit": "us",
})
data = rv.get_json()
assert rv.status_code == 200
assert data["start"] == "2024-01-01 00:00:00"
assert data["end"] == "2024-01-01 01:00:00"
assert len(data["rows"]) == 2
def test_sqlite_integer_time_unit_us(tmp_path: Path) -> None:
sqlite_file = tmp_path / "events.sqlite"
import sqlite3
conn = sqlite3.connect(sqlite_file)
conn.execute("CREATE TABLE visits (visit_time INTEGER, event TEXT)")
big_ts = 13384551652000000
conn.execute("INSERT INTO visits VALUES (?, ?)", (big_ts, "foo"))
conn.commit()
conn.close() # pyright: ignore[reportUnknownMemberType, reportAttributeAcce
ssIssue]
app = server.create_app(sqlite_file)
client = app.test_client()
payload = {
"table": "visits",
"start": "2394-02-20 00:00:00",
"end": "2394-02-21 00:00:00",
"order_by": "visit_time",
"columns": ["visit_time", "event"],
"time_column": "visit_time",
"time_unit": "us",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert len(data["rows"]) == 1
def test_envvar_db(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
csv_file = tmp_path / "custom.csv"
csv_file.write_text("timestamp,event,value,user\n2024-01-01 00:00:00,login,5
,bob\n")
monkeypatch.setenv("SCUBADUCK_DB", str(csv_file))
app = server.create_app()
client = app.test_client()
payload = _make_payload()
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
rows = rv.get_json()["rows"]
assert len(rows) == 1
def test_envvar_db_missing(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> N
one:
missing = tmp_path / "missing.sqlite"
monkeypatch.setenv("SCUBADUCK_DB", str(missing))
with pytest.raises(FileNotFoundError):
server.create_app()
def test_group_by_table() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "table",
"order_by": "user",
"limit": 10,
"columns": ["value"],
"group_by": ["user"],
"aggregate": "Sum",
"show_hits": True,
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
rows = rv.get_json()["rows"]I think there are additional lines past 640, so I'll scroll further.
assert rv.status_code == 200
rows = data["rows"]
from dateutil import parser
timestamps = [parser.parse(r[0]).replace(tzinfo=None) for r in rows]
assert timestamps == sorted(timestamps)
def test_timeseries_limit_applies_to_series() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "timeseries",
"limit": 1,
"order_by": "user",
"group_by": ["user"],
"aggregate": "Count",
"columns": ["value"],
"x_axis": "timestamp",
"granularity": "1 day",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert len(data["rows"]) == 2
assert all(r[1] == "alice" for r in data["rows"])
def test_timeseries_auto_and_fine_buckets() -> None:
app = server.app
client = app.test_client()
def run(gran: str) -> None:
payload = {
"start": "2024-01-01 00:00:00",
"end": "2024-01-02 03:00:00",
"graph_type": "timeseries",
"columns": ["value"],
"x_axis": "timestamp",
"granularity": gran,
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/js
on"
)
data = rv.get_json()
assert rv.status_code == 200
from dateutil import parser
start = parser.parse(cast(str, payload["start"])).replace(tzinfo=None)
buckets = [
parser.parse(cast(str, r[0])).replace(tzinfo=None) for r in data["ro
ws"]
]
assert buckets[0] == start
if len(buckets) > 1:
step = (buckets[1] - buckets[0]).total_seconds()
assert step % data["bucket_size"] == 0
assert any(r[1] != 0 for r in data["rows"])
run("Auto")
run("Fine")
def test_timeseries_string_column_error() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "timeseries",
"limit": 7,
"columns": ["timestamp", "event", "value", "user"],
"x_axis": "timestamp",
"granularity": "1 hour",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 400
assert "Aggregate" in data["error"]
def test_derived_column_basic() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"limit": 10,
"columns": ["timestamp"],
"derived_columns": {"val2": "value * 2"},
"filters": [],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert data["rows"][0][1] == 20
def test_timeseries_derived_column() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "timeseries",
"granularity": "1 hour",
"limit": 7,
"columns": ["value"],
"derived_columns": {"derived_1": "value * 2"},
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
rows = data["rows"]
assert all(r[2] == r[1] * 2 for r in rows)
def test_default_start_end_returned() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"order_by": "timestamp",
"limit": 5,
"columns": ["timestamp"],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert data["start"] == "2024-01-01 00:00:00"
assert data["end"] == "2024-01-02 03:00:00"I'm scrolling down to check the remaining entries below. Almost there!
root@ee1d42b3cdab:/workspace/scubaduck# sed -n '960,1280p' tests/test_server.pyI'll zero in on tests/test_web.py, since that's where the tests are.
def run_query(
page: Any,
url: str,
*,
start: str | None = None,
end: str | None = None,
order_by: str | None = None,
order_dir: str | None = "ASC",
limit: int | None = None,
group_by: list[str] | None = None,
aggregate: str | None = None,
) -> dict[str, Any]:
page.goto(url)
page.wait_for_selector("#order_by option", state="attached")
page.wait_for_selector("#order_dir", state="attached")
page.wait_for_function("window.lastResults !== undefined")
if start is not None:
page.fill("#start", start)
if end is not None:
page.fill("#end", end)
if order_by is not None:
select_value(page, "#order_by", order_by)
if order_dir is not None and order_dir == "DESC":
page.click("#order_dir")
if limit is not None:
page.fill("#limit", str(limit))
if group_by is not None:
select_value(page, "#graph_type", "table")
page.evaluate(
"g => { groupBy.chips = g; groupBy.renderChips(); }",
group_by,
)
if aggregate is not None:
select_value(page, "#graph_type", "table")
select_value(page, "#aggregate", aggregate)
if page.input_value("#graph_type") != "samples":
page.click("text=Columns")
page.wait_for_selector("#column_groups input", state="attached")
if not page.is_checked("#column_groups input[value='value']"):
page.check("#column_groups input[value='value']")
order_col = order_by or page.input_value("#order_by")
if order_col and not page.is_checked(
f"#column_groups input[value='{order_col}']"
):
if page.query_selector(f"#column_groups input[value='{order_col}']")
:
page.check(f"#column_groups input[value='{order_col}']")
page.click("text=View Settings")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
return page.evaluate("window.lastResults")
def test_range_filters(page: Any, server_url: str) -> None:
data = run_query(
page,
server_url,
start="2024-01-02 00:00:00",
end="2024-01-02 04:00:00",
order_by="timestamp",
limit=100,
)
assert len(data["rows"]) == 2
from dateutil import parser
timestamps = [parser.parse(row[0]).replace(tzinfo=None) for row in data["row
s"]]
assert timestamps == [
parser.parse("2024-01-02 00:00:00"),
parser.parse("2024-01-02 03:00:00"),
]
def test_order_by(page: Any, server_url: str) -> None:
data = run_query(
page,
server_url,
start="2024-01-01 00:00:00",
end="2024-01-03 00:00:00",
order_by="value",
order_dir="DESC",
limit=100,
)
values = [row[2] for row in data["rows"]]
assert values == sorted(values, reverse=True)
def test_limit(page: Any, server_url: str) -> None:
data = run_query(
page,
server_url,
start="2024-01-01 00:00:00",
end="2024-01-03 00:00:00",
order_by="timestamp",
limit=2,
)
assert len(data["rows"]) == 2
def test_time_column_dropdown(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#time_column option", state="attached")
options = page.locator("#time_column option").all_inner_texts()
assert "timestamp" in options
assert "value" in options
assert page.input_value("#time_column") == "timestamp"
def test_time_unit_dropdown(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#time_unit", state="attached")
opts = page.locator("#time_unit option").all_inner_texts()
assert "ms" in opts
assert page.input_value("#time_unit") == "s"
def test_x_axis_default_entry(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#graph_type", state="attached")
select_value(page, "#graph_type", "timeseries")
page.wait_for_selector("#x_axis option", state="attached")
options = page.locator("#x_axis option").all_inner_texts()
assert "(default)" in options
assert page.input_value("#x_axis") == ""
def test_simple_filter(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
page.click("text=Add Filter")
filter_el = page.query_selector("#filters .filter:last-child")
assert filter_el
page.evaluate(
"arg => setSelectValue(arg.el.querySelector('.f-col'), arg.val)",
{"el": filter_el, "val": "user"},
)
val_input = filter_el.query_selector(".f-val")
val_input.click()
page.keyboard.type("alice")
page.keyboard.press("Enter")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
data = page.evaluate("window.lastResults")
assert len(data["rows"]) == 2
assert all(row[3] == "alice" for row in data["rows"])
def test_default_filter_and_layout(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
count = page.evaluate("document.querySelectorAll('#filters .filter').length"
)
assert count == 1
last_is_button = page.evaluate(
"document.querySelector('#filters').lastElementChild.id === 'add_filter'
"
)
assert last_is_button
position = page.evaluate(
"getComputedStyle(document.querySelector('#filters .filter button.remove
')).position"
)
assert position == "static"
def test_filter_remove_alignment(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
diff = page.evaluate(
"() => { const r=document.querySelector('#filters .filter-row').getBound
ingClientRect(); const x=document.querySelector('#filters .filter-row button.rem
ove').getBoundingClientRect(); return Math.abs(r.right - x.right); }"
)
assert diff <= 1
def test_header_and_tabs(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
header = page.text_content("#header")
assert "sample.csv" in header
assert "events" in header
assert page.is_visible("#settings")
assert page.is_hidden("#columns")
page.click("text=Columns")
assert page.is_visible("#columns")
cols = [c.strip() for c in page.locator("#column_groups li").all_inner_texts
()]
assert "timestamp" in cols
assert "event" in cols
page.click("text=View Settings")
assert page.is_visible("#settings")I’ll check the remaining files to gather everything needed.
view_overflow = page.evaluate(
"getComputedStyle(document.querySelector('#view')).overflowY"
)
assert sidebar_overflow == "auto"
assert view_overflow == "auto"
def test_graph_type_table_fields(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#graph_type", state="attached")
select_value(page, "#graph_type", "table")
assert page.is_visible("#group_by_field")
assert page.is_visible("#aggregate_field")
assert page.is_visible("#show_hits_field")
page.click("text=Columns")
assert not page.is_visible("text=Strings:")
def test_graph_type_timeseries_fields(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#graph_type", state="attached")
select_value(page, "#graph_type", "timeseries")
assert page.is_visible("#group_by_field")
assert page.is_visible("#aggregate_field")
assert page.is_visible("#x_axis_field")
assert page.is_visible("#granularity_field")
assert page.is_visible("#fill_field")
def test_limit_persists_per_chart_type(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#graph_type", state="attached")
assert page.input_value("#limit") == "100"
select_value(page, "#graph_type", "timeseries")
assert page.input_value("#limit") == "7"
select_value(page, "#graph_type", "samples")
assert page.input_value("#limit") == "100"
def test_columns_persist_per_chart_type(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#graph_type", state="attached")
page.click("text=Columns")
page.wait_for_selector("#column_groups input", state="attached")
page.uncheck("#column_groups input[value='value']")
select_value(page, "#graph_type", "timeseries")
count = page.evaluate(
"document.querySelectorAll('#column_groups input:checked').length"
)
assert count == 0
select_value(page, "#graph_type", "samples")
count = page.evaluate(
"document.querySelectorAll('#column_groups input:checked').length"
)
assert count == 3
def test_timeseries_default_query(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#graph_type", state="attached")
select_value(page, "#graph_type", "timeseries")
page.click("text=Columns")
page.check("#column_groups input[value='value']")
page.click("text=View Settings")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
data = page.evaluate("window.lastResults")
assert "error" not in data
assert page.is_visible("#chart")
page.click("text=Columns")
assert not page.is_checked("#column_groups input[value='timestamp']")
def test_timeseries_single_bucket(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#graph_type", state="attached")
page.fill("#start", "2024-01-01 00:00:00")
page.fill("#end", "2024-01-01 00:00:00")
select_value(page, "#graph_type", "timeseries")
page.click("text=Columns")
page.check("#column_groups input[value='value']")
page.click("text=View Settings")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
path = page.get_attribute("#chart path", "d")
assert path is not None and "NaN" not in path
def test_timeseries_fill_options(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#graph_type", state="attached")
page.fill("#start", "2024-01-01 00:00:00")
page.fill("#end", "2024-01-02 03:00:00")
select_value(page, "#graph_type", "timeseries")
page.click("text=Columns")
page.check("#column_groups input[value='value']")
page.click("text=View Settings")
select_value(page, "#granularity", "1 hour")
select_value(page, "#fill", "0")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
path_zero = page.get_attribute("#chart path", "d")
assert path_zero is not None and path_zero.count("L") > 20
select_value(page, "#fill", "connect")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
path_conn = page.get_attribute("#chart path", "d")
assert path_conn is not None and path_conn.count("M") == 1
select_value(page, "#fill", "blank")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
path_blank = page.get_attribute("#chart path", "d")
assert path_blank is not None and path_blank.count("M") > 1
def test_timeseries_hover_highlight(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#graph_type", state="attached")
select_value(page, "#graph_type", "timeseries")
page.click("text=Columns")
page.check("#column_groups input[value='value']")
page.click("text=View Settings")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
page.wait_for_selector("#chart path", state="attached")
path_el = page.query_selector("#chart path")
assert path_el
page.evaluate(
"el => el.dispatchEvent(new MouseEvent('mouseenter', {bubbles: true}))",
path_el,
)
width = page.evaluate(
"getComputedStyle(document.querySelector('#chart path')).strokeWidth"
)
assert "2.5" in width
color = page.evaluate(
"getComputedStyle(document.querySelector('#legend .legend-item')).backgr
oundColor"
)
assert "221, 221, 221" in color
def test_timeseries_crosshair(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#graph_type", state="attached")
select_value(page, "#graph_type", "timeseries")
page.click("text=Columns")
page.check("#column_groups input[value='value']")
page.click("text=View Settings")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
page.wait_for_selector("#chart path", state="attached")
page.eval_on_selector(
"#chart",
"el => { const r = el.getBoundingClientRect(); el.dispatchEvent(new Mous
eEvent('mousemove', {clientX: r.left + r.width/2, clientY: r.top + r.height/2, b
ubbles: true})); }",
)
line_display = page.evaluate(
"document.getElementById('crosshair_line').style.display"
)
assert line_display != "none"
count = page.eval_on_selector_all("#crosshair_dots circle", "els => els.leng
th")
assert count > 0
page.eval_on_selector(
"#chart",
"el => el.dispatchEvent(new MouseEvent('mouseleave', {bubbles: true}))",
)
line_display = page.evaluate(
"document.getElementById('crosshair_line').style.display"
)
assert line_display == "none"
def test_timeseries_crosshair_freeze(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#graph_type", state="attached")
select_value(page, "#graph_type", "timeseries")
page.click("text=Columns")
page.check("#column_groups input[value='value']")
page.click("text=View Settings")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
page.wait_for_selector("#chart path", state="attached")I’m scrolling to around line 400 to check the latest changes.
pos1 = page.evaluate("document.getElementById('crosshair_line').getAttribute
('x1')")
page.eval_on_selector(
"#chart",
"el => { const r = el.getBoundingClientRect(); el.dispatchEvent(new Mous
eEvent('mousemove', {clientX: r.left + r.width/4, clientY: r.top + r.height/2, b
ubbles: true})); }",
)
pos2 = page.evaluate("document.getElementById('crosshair_line').getAttribute
('x1')")
assert pos1 == pos2
page.eval_on_selector(
"#chart",
"el => el.dispatchEvent(new MouseEvent('mouseleave', {bubbles: true}))",
)
line_display = page.evaluate(
"document.getElementById('crosshair_line').style.display"
)
assert line_display != "none"
page.eval_on_selector(
"#chart",
"el => { const r = el.getBoundingClientRect(); el.dispatchEvent(new Mous
eEvent('click', {clientX: r.left + r.width/2, clientY: r.top + r.height/2, bubbl
es: true})); }",
)
line_display = page.evaluate(
"document.getElementById('crosshair_line').style.display"
)
assert line_display == "none"
def test_timeseries_auto_timezone(browser: Any, server_url: str) -> None:
context = browser.new_context(timezone_id="America/New_York")
page = context.new_page()
page.goto(server_url)
page.wait_for_selector("#graph_type", state="attached")
select_value(page, "#graph_type", "timeseries")
page.click("text=Columns")
page.check("#column_groups input[value='value']")
page.click("text=View Settings")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
path = page.get_attribute("#chart path", "d")
context.close()
assert path is not None
coords = [float(p.split(" ")[1]) for p in path.replace("M", "L").split("L")[
1:]]
assert max(coords) > min(coords)
def test_timeseries_multi_series(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#graph_type", state="attached")
select_value(page, "#graph_type", "timeseries")
page.click("text=Columns")
page.check("#column_groups input[value='value']")
page.click("text=Add Derived")
expr = page.query_selector("#derived_list .derived textarea")
assert expr
name_inp = page.query_selector("#derived_list .derived .d-name")
assert name_inp
name_inp.fill("value_2")
expr.fill("value * 2")
page.click("text=View Settings")
page.fill("#start", "2024-01-01 00:00:00")
page.fill("#end", "2024-01-03 00:00:00")
select_value(page, "#granularity", "1 hour")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
count = page.eval_on_selector_all("#chart path", "els => els.length")
assert count == 2
def test_help_and_alignment(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
titles = page.evaluate(
"Array.from(document.querySelectorAll('#settings .help')).map(e => e.tit
le)"
)
assert any("start/end of the time range" in t for t in titles)
text_align = page.evaluate(
"getComputedStyle(document.querySelector('#settings label')).textAlign"
)
assert text_align == "right"
def test_table_sorting(page: Any, server_url: str) -> None:
run_query(
page,
server_url,
start="2024-01-01 00:00:00",
end="2024-01-03 00:00:00",
order_by="timestamp",
order_dir="ASC",
limit=100,
)
# header alignment
align = page.evaluate(
"getComputedStyle(document.querySelector('#results th')).textAlign"
)
assert align == "right"
header = page.locator("#results th").nth(3)
def values() -> list[str]:
return page.locator("#results td:nth-child(4)").all_inner_texts()
orig_rows = values()
assert orig_rows == ["alice", "bob", "alice", "charlie"]
first_sql = page.evaluate("window.lastResults.sql")
header.click()
assert values() == sorted(orig_rows)
assert header.inner_text().endswith("▲")
color = page.evaluate(
"getComputedStyle(document.querySelector('#results th:nth-child(4)')).co
lor"
)
assert "0, 0, 255" in color
...
assert f
page.evaluate(
"arg => setSelectValue(arg.el.querySelector('.f-col'), arg.val)",
{"el": f, "val": "user"},
)
inp = f.query_selector(".f-val")
inp.click()
page.keyboard.type("alice")
page.keyboard.press("Enter")
inp.click()
page.keyboard.type("bob")
page.keyboard.press("Enter")
f.query_selector(".chip-copy").click()
assert page.evaluate("navigator.clipboard._data") == "alice,bob"
page.evaluate(
"var f=document.querySelector('#filters .filter:last-child'); f.chips=[]
; f.querySelectorAll('.chip').forEach(c=>c.remove())"
)
page.wait_for_selector("#filters .chip", state="detached")
inp.click()
page.evaluate(
"var dt=new DataTransfer(); dt.setData('text/plain','alice,bob'); var e=
new ClipboardEvent('paste',{clipboardData:dt}); document.querySelector('#filters
.filter:last-child .f-val').dispatchEvent(e);"
)
chips = page.evaluate(
"Array.from(document.querySelectorAll('#filters .filter:last-child .chip
')).map(c => c.firstChild.textContent)"
)
assert chips[:2] == ["alice", "bob"]
page.evaluate(
"var f=document.querySelector('#filters .filter:last-child'); f.chips=[]
; f.querySelectorAll('.chip').forEach(c=>c.remove())"
)
page.wait_for_selector("#filters .chip", state="detached")
inp.click()
page.evaluate(
"var dt=new DataTransfer(); dt.setData('text/plain','alice,bob'); var e=
new ClipboardEvent('paste',{clipboardData:dt}); Object.defineProperty(e,'shiftKe
y',{value:true}); document.querySelector('#filters .filter:last-child .f-val').d
ispatchEvent(e);"
)
chips = page.evaluate(
"Array.from(document.querySelectorAll('#filters .filter:last-child .chip
')).map(c => c.firstChild.textContent)"
)
assert chips[-1] == "alice,bob"
def test_chip_dropdown_hides_on_outside_click(page: Any, server_url: str) -> Non
e:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
page.click("text=Add Filter")
f = page.query_selector("#filters .filter:last-child")
assert f
page.evaluate(
"arg => setSelectValue(arg.el.querySelector('.f-col'), arg.val)",
{"el": f, "val": "user"},
)
inp = f.query_selector(".f-val")
inp.click()
page.wait_for_selector("#filters .filter:last-child .chip-dropdown div")
page.click("#header")
page.wait_for_selector("#filters .filter:last-child .chip-dropdown", state="
hidden")
def test_chip_input_no_outline(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
page.click("text=Add Filter")
inp = page.query_selector("#filters .filter:last-child .f-val")
assert inp
inp.click()
outline = page.evaluate(
"getComputedStyle(document.querySelector('#filters .filter:last-child .f
-val')).outlineStyle"
)
assert outline == "none"
def test_chip_enter_keeps_focus(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
page.click("text=Add Filter")
f = page.query_selector("#filters .filter:last-child")
assert f
page.evaluate(
"arg => setSelectValue(arg.el.querySelector('.f-col'), arg.val)",
{"el": f, "val": "user"},
)
inp = f.query_selector(".f-val")
inp.click()
page.wait_for_selector("#filters .filter:last-child .chip-dropdown")
page.keyboard.type("alice")
page.keyboard.press("Enter")
focused = page.evaluate(
"document.activeElement === document.querySelector('#filters .filter:las
t-child .f-val')"
)
assert focused
visible = page.evaluate(
"getComputedStyle(document.querySelector('#filters .filter:last-child .c
hip-dropdown')).display"
)
assert visible == "none"
def test_chip_delete_keeps_focus(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
page.click("text=Add Filter")
f = page.query_selector("#filters .filter:last-child")
assert f
page.evaluate(
"arg => setSelectValue(arg.el.querySelector('.f-col'), arg.val)",
{"el": f, "val": "user"},
)
inp = f.query_selector(".f-val")
inp.click()
page.wait_for_selector("#filters .filter:last-child .chip-dropdown")
page.keyboard.type("alice")
page.keyboard.press("Enter")f = page.query_selector("#filters .filter:last-child")
assert f
page.evaluate(
"arg => setSelectValue(arg.el.querySelector('.f-col'), arg.val)",
{"el": f, "val": "user"},
)
inp = f.query_selector(".f-val")
inp.click()
page.wait_for_selector("#filters .filter:last-child .chip-dropdown")
page.keyboard.type("ali")
page.wait_for_selector(
"#filters .filter:last-child .chip-dropdown div:text('alice')"
)
page.click("#filters .filter:last-child .chip-dropdown div:text('alice')")
focused = page.evaluate(
"document.activeElement === document.querySelector('#filters .filter:las
t-child .f-val')"
)
assert not focused
visible = page.evaluate(
"getComputedStyle(document.querySelector('#filters .filter:last-child .c
hip-dropdown')).display"
)
assert visible == "none"
def test_chip_dropdown_hides_on_column_click(page: Any, server_url: str) -> None
:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
page.click("text=Add Filter")
f = page.query_selector("#filters .filter:last-child")
assert f
page.evaluate(
"arg => setSelectValue(arg.el.querySelector('.f-col'), arg.val)",
{"el": f, "val": "user"},
)
inp = f.query_selector(".f-val")
inp.click()
page.wait_for_selector("#filters .filter:last-child .chip-dropdown div")
f.query_selector(".f-col + .dropdown-display").click()
page.wait_for_selector("#filters .filter:last-child .chip-dropdown", state="
hidden")
def test_chip_backspace_keeps_dropdown(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
page.click("text=Add Filter")
f = page.query_selector("#filters .filter:last-child")
assert f
page.evaluate(
"arg => setSelectValue(arg.el.querySelector('.f-col'), arg.val)",
{"el": f, "val": "user"},
)
inp = f.query_selector(".f-val")
inp.click()
page.keyboard.type("alice")
page.keyboard.press("Enter")
page.keyboard.type("b")
page.wait_for_selector("#filters .filter:last-child .chip-dropdown div")
page.keyboard.press("Backspace")
page.wait_for_function(
"document.querySelector('#filters .filter:last-child .f-val').value ===
''"
)
focused = page.evaluate(
"document.activeElement === document.querySelector('#filters .filter:las
t-child .f-val')"
)
assert focused
visible = page.evaluate(
"getComputedStyle(document.querySelector('#filters .filter:last-child .c
hip-dropdown')).display"
)
assert visible == "block"
def test_chip_duplicate_toggles(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
page.click("text=Add Filter")
f = page.query_selector("#filters .filter:last-child")
assert f
page.evaluate(
"arg => setSelectValue(arg.el.querySelector('.f-col'), arg.val)",
{"el": f, "val": "user"},
)
inp = f.query_selector(".f-val")
inp.click()
page.keyboard.type("alice")
page.keyboard.press("Enter")
chips = page.evaluate(
"Array.from(document.querySelectorAll('#filters .filter:last-child .chip
')).map(c => c.firstChild.textContent)"
)
assert chips == ["alice"]
inp.click()
page.keyboard.type("alice")
page.keyboard.press("Enter")
chips = page.evaluate(
"Array.from(document.querySelectorAll('#filters .filter:last-child .chip
')).map(c => c.firstChild.textContent)"
)
assert chips == []
def test_table_enhancements(page: Any, server_url: str) -> None:
run_query(
page,
server_url,
start="2024-01-01 00:00:00",
end="2024-01-03 00:00:00",
order_by="timestamp",
limit=10,
)
border = page.evaluate(
"getComputedStyle(document.querySelector('#results td')).borderStyle"
)
assert border == "solid"
color1 = page.evaluate(
"getComputedStyle(document.querySelector('#results tr:nth-child(2) td'))
.backgroundColor"
...
erStyle"
)
assert border == "none"
def test_table_group_by_query(page: Any, server_url: str) -> None:
data = run_query(
page,
server_url,
start="2024-01-01 00:00:00",
end="2024-01-03 00:00:00",
order_by="timestamp",
limit=100,
group_by=["user"],
aggregate="Count",
)
assert "error" not in data
assert len(data["rows"]) == 3
def test_table_avg_no_group_by(page: Any, server_url: str) -> None:
data = run_query(
page,
server_url,
aggregate="Avg",
)
assert len(data["rows"]) == 1
row = data["rows"][0]
assert row[0] == 4
from dateutil import parser
ts = parser.parse(row[1]).replace(tzinfo=None)
assert ts == parser.parse("2024-01-01 13:00:00")
assert row[2] == 25
def test_table_headers_show_aggregate(page: Any, server_url: str) -> None:
run_query(
page,
server_url,
aggregate="Avg",
)
headers = page.locator("#results th").all_inner_texts()
assert "Hits" in headers
assert "timestamp (avg)" in headers
assert "value (avg)" in headers
def test_format_number_function(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
vals = page.evaluate(
"() => [formatNumber(815210), formatNumber(999.999), formatNumber(0.0004
), formatNumber(0)]"
)
assert vals == ["815.21 K", "999.999", "0.000", "0"]
def test_numeric_cell_nowrap(page: Any, server_url: str) -> None:
run_query(page, server_url, limit=10)
whitespace = page.evaluate(
"getComputedStyle(document.querySelector('#results td:nth-child(3)')).wh
iteSpace"
)
assert whitespace == "nowrap"
def test_derived_column_query(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
page.click("text=Columns")
page.click("text=Add Derived")
expr = page.query_selector("#derived_list .derived textarea")
assert expr
expr.fill("value * 2")
page.click("text=View Settings")
page.fill("#start", "2024-01-01 00:00:00")
page.fill("#end", "2024-01-03 00:00:00")
page.fill("#limit", "10")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
data = page.evaluate("window.lastResults")
assert data["rows"][0][-1] == 20
def test_derived_column_remove(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
page.click("text=Columns")
page.click("text=Add Derived")
assert page.query_selector("#derived_list .derived button.remove")
page.click("#derived_list .derived button.remove")
count = page.evaluate("document.querySelectorAll('#derived_list .derived').l
ength")
assert count == 0
def test_sql_query_display(page: Any, server_url: str) -> None:
data = run_query(
page,
server_url,
start="2024-01-01 00:00:00",
end="2024-01-02 00:00:00",
order_by="timestamp",
limit=10,
)
sql = data["sql"]
displayed = page.text_content("#sql_query")
assert displayed is not None
assert displayed.strip() == sql
def test_timeseries_resize(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#graph_type", state="attached")
select_value(page, "#graph_type", "timeseries")
page.click("text=Columns")
page.check("#column_groups input[value='value']")
page.click("text=View Settings")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
page.wait_for_selector("#chart path", state="attached")I’m going further to confirm the presence of time_column.
root@ee1d42b3cdab:/workspace/scubaduck# grep -n "time_column" -n tests/test_web.
py
111:def test_time_column_dropdown(page: Any, server_url: str) -> None:
113: page.wait_for_selector("#time_column option", state="attached")
114: options = page.locator("#time_column option").all_inner_texts()
117: assert page.input_value("#time_column") == "timestamp"Only one test is referencing time_column. I'll check index.html for modifications.
tml
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<title>ScubaDuck</title>
<style>
body { margin: 0; font-family: sans-serif; height: 100vh; display: flex; fle
x-direction: column; }
#header { padding: 10px; font-weight: bold; border-bottom: 1px solid #ccc; }
#content { flex: 1; display: flex; height: calc(100vh - 42px); overflow: hid
den; }
#sidebar { width: 450px; padding: 10px; border-right: 3px solid #ccc; overfl
ow-y: auto; display: flex; flex-direction: column; box-sizing: border-box; }
#sidebar-resizer { width: 5px; cursor: col-resize; background: #ccc; }
#view { flex: 1; padding: 10px; overflow-y: auto; overflow-x: auto; }
.field { display: flex; align-items: center; margin-bottom: 10px; }
.field label { width: 80px; text-align: right; margin-right: 5px; }
.help { margin-left: 4px; cursor: help; }
.rel-btn { margin-left: 4px; }
#tabs { display: flex; align-items: center; margin-bottom: 10px; }
#tabs .tab { margin-right: 5px; background: none; border: 1px solid #ccc; pa
dding: 4px 8px; cursor: pointer; width: 120px; text-align: center; box-sizing: b
order-box; }
#tabs .tab.active { background: #eee; font-weight: bold; }
#dive { margin-left: auto; background: green; color: white; border: none; pa
dding: 5px 10px; cursor: pointer; }
.tab-content { display: none; }
.tab-content.active { display: block; }
#filter_list { display: flex; flex-direction: column; }
#filters .filter {
border: 1px solid #ccc;
padding: 5px;
margin-bottom: 5px;
position: relative;
display: flex;
flex-direction: column;
}
#derived_columns .derived {
border: 1px solid #ccc;
padding: 5px;
margin-bottom: 5px;
display: flex;
flex-direction: column;
}
#derived_columns .derived-row {
display: flex;
margin-bottom: 5px;
}
#derived_columns .derived-row input[type="text"] {
margin-left: 5px;
flex: 1;
}
#derived_columns .derived-row button.remove {
margin-left: 5px;
width: 20px;
flex: 0 0 auto;
padding: 0;
text-align: center;
line-height: 1;
}
#derived_columns textarea {
width: 100%;
box-sizing: border-box;
}
#filters .filter-row { display: flex; margin-bottom: 5px; }
#filters .filter-row .f-col { flex: 1; }
#filters .filter-row .f-op {
margin-left: 5px;
width: fit-content;
flex: 0 0 auto;
}
.chip-input input {
border: none;
flex: 1;
min-width: 60px;
margin: 2px;
outline: none;
}
.chip-box { position: relative; }
.chip-input { display: flex; flex-wrap: wrap; border: 1px solid #ccc; paddin
g: 2px; min-height: 24px; }
.chip { background: #eee; border: 1px solid #999; padding: 2px 4px; margin:
2px; border-radius: 3px; display: flex; align-items: center; }
.chip .x { margin-left: 4px; cursor: pointer; }
.chip-copy { margin-left: 4px; cursor: pointer; background: none; border: no
ne; }
.chip-dropdown { position: absolute; left: 0; right: 0; top: 100%; backgroun
d: white; border: 1px solid #ccc; max-height: 120px; overflow-y: auto; z-index:
10; display: none; }
.chip-dropdown div { padding: 2px 4px; cursor: pointer; }
.chip-dropdown div.highlight { background: #bde4ff; }
.rel-box { position: relative; display: flex; }
.rel-dropdown { position: absolute; left: 0; right: 0; top: 100%; background
: white; border: 1px solid #ccc; z-index: 10; display: none; }
.rel-dropdown div { padding: 2px 4px; cursor: pointer; }
.rel-dropdown div:hover { background: #bde4ff; }
.dropdown { position: relative; display: inline-block; }
.dropdown-display {
border: 1px solid #ccc;
padding: 2px 18px 2px 4px;
cursor: pointer;
min-width: 80px;
position: relative;
}
.dropdown-display::after {
content: '\25BC';
position: absolute;
right: 4px;
pointer-events: none;
}
.dropdown-menu { position: absolute; left: 0; right: 0; top: 100%; backgroun
d: white; border: 1px solid #ccc; z-index: 10; max-height: 160px; overflow-y: au
to; display: none; }
.dropdown-menu input { width: 100%; box-sizing: border-box; padding: 2px 4px
; border: none; border-bottom: 1px solid #ccc; }
.dropdown-menu div { padding: 2px 4px; cursor: pointer; }
.dropdown-menu div.selected { background: #bde4ff; }
.dropdown-menu .option:hover { background: #eee; }
.dropdown-menu input::placeholder { color: #999; }
#filters .filter button.remove {
margin-left: 5px;
width: 20px;
flex: 0 0 auto;
padding: 0;
text-align: center;
line-height: 1;
}
#filters h4 { margin: 0 0 5px 0; }
table { border-collapse: collapse; min-width: 100%; }
th, td { border: 1px solid #ccc; padding: 4px; box-sizing: border-box; }
td.numeric { white-space: nowrap; }
th { text-align: left; cursor: pointer; position: relative; }
th.sorted { color: blue; }
tr:nth-child(even) td { background: #f9f9f9; }
tr.selected td { background: #bde4ff !important; }
tr:hover:not(.selected) td { background: #eee; }
#column_actions {
text-align: right;
margin-bottom: 5px;
}
#column_actions a {
margin-left: 5px;
}
.col-group-header {
overflow: hidden;
}
.col-group-header .links {
float: right;
}
.col-group-header .links a {
margin-left: 5px;
}
#ts-container {
display: flex;
}
#legend {
width: 150px;
flex: 0 0 150px;
margin-right: 10px;
overflow-y: auto;
}
.legend-group {
margin-bottom: 4px;
}
.legend-header {
font-weight: normal;
}
.legend-item {
display: flex;
justify-content: space-between;
padding-left: 8px;
}
.legend-value {
margin-left: 4px;
}
#chart-wrapper {
flex: 1;
}
.legend-item.highlight {
background: #ddd;
}
#legend .drill-links h4 {
margin: 10px 0 4px 0;
}
#legend .drill-links a {
display: block;
margin-left: 8px;
}
#chart text.tick-label {
font-size: 10px;
user-select: none;
}
#chart text.tick-label.rotated {
text-anchor: end;
}
#chart text.y-tick-label {
font-size: 10px;
user-select: none;
}
#chart line.grid {
stroke: #ccc;
}
/* Column resizer removed */
</style>
</head>
<body>
<div id="header">sample.csv - <select id="table"></select> <select id="graph_t
ype"><option value="samples">Samples</option><option value="table">Table</option
><option value="timeseries">Time Series</option></select></div>
<div id="content">
<div id="sidebar">
<div id="tabs">
<button class="tab active" data-tab="settings">View Settings</button>
<button class="tab" data-tab="columns">Columns</button>
<button id="dive" onclick="dive()">Dive</button>
</div>
<div id="settings" class="tab-content active">I'm going to keep scrolling from around line 200 towards line 400.
root@ee1d42b3cdab:/workspace/scubaduck# sed -n '200,400p' scubaduck/static/index
.html
<div id="settings" class="tab-content active">
<div class="field">
<label>Time Column</label>
<select id="time_column"></select>
<select id="time_unit" style="margin-left:4px">
<option value="s">s</option>
<option value="ms">ms</option>
<option value="us">us</option>
<option value="ns">ns</option>
</select>
</div>
<div class="field">
<label>Start<span class="help" title="Sets the start/end of the time r
ange to query. Can be any kind of datetime string. For example: 'April 23, 2014'
or 'yesterday'.">[?]</span></label>
<div class="rel-box">
<input id="start" type="text" />
<button type="button" class="rel-btn" data-target="start-select">	
660;</button>
<div id="start-select" class="rel-dropdown" data-input="start">
<div data-value="-1 hour">-1 hour</div>
<div data-value="-3 hours">-3 hours</div>
<div data-value="-12 hours">-12 hours</div>
<div data-value="-1 day">-1 day</div>
<div data-value="-3 days">-3 days</div>
<div data-value="-1 week">-1 week</div>
<div data-value="-1 fortnight">-1 fortnight</div>
<div data-value="-30 days">-30 days</div>
<div data-value="-90 days">-90 days</div>
</div>
</div>
</div>
<div class="field">
<label>End<span class="help" title="Sets the start/end of the time ran
ge to query. Can be any kind of datetime string. For example: 'April 23, 2014' o
r 'yesterday'.">[?]</span></label>
<div class="rel-box">
<input id="end" type="text" />
<button type="button" class="rel-btn" data-target="end-select">φ
0;</button>
<div id="end-select" class="rel-dropdown" data-input="end">
<div data-value="now">now</div>
<div data-value="-1 hour">-1 hour</div>
<div data-value="-3 hours">-3 hours</div>
<div data-value="-12 hours">-12 hours</div>
<div data-value="-1 day">-1 day</div>
<div data-value="-3 days">-3 days</div>
<div data-value="-1 week">-1 week</div>
<div data-value="-1 fortnight">-1 fortnight</div>
<div data-value="-30 days">-30 days</div>
<div data-value="-90 days">-90 days</div>
</div>
</div>
</div>
<div class="field">
<label>Order By<span class="help" title="Choose a column to sort resul
ts by.">[?]</span></label>
<select id="order_by"></select>
<button id="order_dir" type="button">ASC \u25B2</button>
</div>
<div class="field">
<label>Limit<span class="help" title="Choose the maximum number of res
ults to show in the chart after any aggregations have been applied. For example,
a limit of 10 will show no more than 10 rows for a table, etc.">[?]</span></lab
el>
<input id="limit" type="number" value="100" />
</div>
<div id="x_axis_field" class="field" style="display:none;">
<label>X-axis</label>
<select id="x_axis"></select>
</div>
<div id="granularity_field" class="field" style="display:none;">
<label>Granularity</label>
<select id="granularity">
<option>Auto</option>
<option>Fine</option>
<option>1 second</option>
<option>5 seconds</option>
<option>10 seconds</option>
<option>30 seconds</option>
<option>1 minute</option>
<option>4 minutes</option>
<option>5 minutes</option>
<option>10 minutes</option>
<option>15 minutes</option>
<option>30 minutes</option>
<option>1 hour</option>
<option>3 hours</option>
<option>6 hours</option>
<option>1 day</option>
<option>1 week</option>
<option>30 days</option>
</select>
</div>
<div id="fill_field" class="field" style="display:none;">
<label>Fill Missing Buckets</label>
<select id="fill">
<option value="0">Fill with 0 (Per Series)</option>
<option value="connect">Connect (Per Series)</option>
<option value="blank">Leave blank</option>
</select>
</div>
<div id="group_by_field" class="field" style="display:none;">
<label>Group By</label>
<div class="chip-box">
<div class="chip-input">
<input id="group_by" class="f-val" type="text">
<button type="button" class="chip-copy">⎘</button>
</div>
<div class="chip-dropdown"></div>
</div>
</div>
<div id="aggregate_field" class="field" style="display:none;">
<label>Aggregate</label>
<select id="aggregate">
<option>Avg</option>
<option>Count</option>
<option>Sum</option>
<option>Min</option>
<option>Max</option>
<option>Count Distinct</option>
<option>p5</option>
<option>p25</option>
<option>p50</option>
<option>p70</option>
<option>p75</option>
<option>p90</option>
<option>p95</option>
<option>p99</option>
<option>p99.9</option>
<option>p99.99</option>
</select>
</div>
<div id="show_hits_field" class="field" style="display:none;">
<label>Show Hits</label>
<input id="show_hits" type="checkbox" checked>
</div>
<div id="filters">
<h4>Filters<span class="help" title="You can create as many filters as
you want. You can either write a filter using a UI or manual SQL. In the UI, fi
lter consists of a column name, a relation (e.g., =, !=, <, >) and then a text f
ield. The text field is a token input. It accepts multiple tokens for = relation
, in which case we match using an OR for all options.">[?]</span></h4>
<div id="filter_list"></div>
<button id="add_filter" type="button" onclick="addFilter()">Add Filter
</button>
</div>
<div id="query_info" style="margin-top:10px;"></div>
</div>
<div id="columns" class="tab-content">
<div id="column_actions">
<a id="columns_all" href="#">All</a>
<a id="columns_none" href="#">None</a>
</div>
<div id="column_groups"></div>
<div id="derived_columns">
<h4>Derived Columns</h4>
<div id="derived_list"></div>
<button id="add_derived" type="button" onclick="addDerived()">Add Deri
ved</button>
</div>
</div>
</div>
<div id="sidebar-resizer"></div>
<div id="view">
<table id="results"></table>
</div>
</div>
<script src="/js/chip_input.js"></script>
<script src="/js/timeseries_chart.js"></script>
<script>
const allColumns = [];
const baseColumns = [];
const columnTypes = {};
const stringColumns = [];
const baseStringColumns = [];
const integerColumns = [];
const baseIntegerColumns = [];
const timeColumns = [];
const baseTimeColumns = [];
const timeColumnOptions = [];
const baseTimeColumnOptions = [];
const derivedColumns = [];
let selectedColumns = [];
let displayType = 'samples';
let groupBy = {chips: [], addChip: () => {}, renderChips: () => {}};
let defaultTimeColumn = '';
const limitInput = document.getElementById('limit');
const limitValues = {
samples: parseInt(limitInput.value, 10),
table: parseInt(limitInput.value, 10),
timeseries: 7
};
const columnValues = {
samples: [],
table: [],
timeseries: []
};
limitInput.addEventListener('input', () => {
limitValues[displayType] = parseInt(limitInput.value, 10);
limitInput.dataset.setByUser = '1';
});
function initDropdown(select) {
const wrapper = document.createElement('div');
wrapper.className = 'dropdown';
if (select.classList.contains('f-col')) {
wrapper.classList.add('f-col');
}
select.parentNode.insertBefore(wrapper, select);
wrapper.appendChild(select);
select.style.display = 'none';
const disp = document.createElement('div');
disp.className = 'dropdown-display';
function updateDisplay() {
const opt = select.options[select.selectedIndex];
disp.textContent = opt ? opt.textContent : '';root@ee1d42b3cdab:/workspace/scubaduck# sed -n '400,800p' scubaduck/static/index
.html
disp.textContent = opt ? opt.textContent : '';
}
updateDisplay();
wrapper.appendChild(disp);
const menu = document.createElement('div');
menu.className = 'dropdown-menu';
const search = document.createElement('input');
search.placeholder = 'Search';
menu.appendChild(search);
const list = document.createElement('div');
menu.appendChild(list);
wrapper.appendChild(menu);
function close() {
menu.style.display = 'none';
}
function open() {
renderOptions();
menu.style.display = 'block';
search.focus();
}
disp.addEventListener('click', () => {
if (menu.style.display === 'block') {
close();
} else {
open();
}
});
document.addEventListener('click', e => {
if (!wrapper.contains(e.target)) {
close();
}
});
function renderOptions() {
const q = search.value.toLowerCase();
list.innerHTML = '';
Array.from(select.options).forEach(o => {
if (!o.textContent.toLowerCase().includes(q)) return;
const div = document.createElement('div');
div.className = 'option';
if (q) {
const text = o.textContent;
const idx = text.toLowerCase().indexOf(q);
if (idx !== -1) {
div.innerHTML =
text.slice(0, idx) +
'<u>' +
text.slice(idx, idx + q.length) +
'</u>' +
text.slice(idx + q.length);
} else {
div.textContent = text;
}
} else {
div.textContent = o.textContent;
}
if (o.value === select.value) div.classList.add('selected');
div.addEventListener('mousedown', evt => {
evt.preventDefault();
select.value = o.value;
select.dispatchEvent(new Event('change'));
updateDisplay();
close();
});
list.appendChild(div);
});
}
search.addEventListener('input', renderOptions);
select.addEventListener('change', updateDisplay);
}
// Sidebar resizing
const sidebar = document.getElementById('sidebar');
const sidebarResizer = document.getElementById('sidebar-resizer');
let sidebarWidth = parseInt(localStorage.getItem('sidebarWidth') || 450, 10);
sidebar.style.width = sidebarWidth + 'px';
let sidebarResize = false;
function startSidebarDrag(e) {
e.preventDefault();
sidebarResize = true;
document.addEventListener('mousemove', onSidebarDrag);
document.addEventListener('mouseup', stopSidebarDrag);
}
function onSidebarDrag(e) {
if (!sidebarResize) return;
sidebarWidth = Math.max(200, e.clientX - sidebar.getBoundingClientRect().left)
;
sidebar.style.width = sidebarWidth + 'px';
}
function stopSidebarDrag() {
document.removeEventListener('mousemove', onSidebarDrag);
document.removeEventListener('mouseup', stopSidebarDrag);
sidebarResize = false;
localStorage.setItem('sidebarWidth', sidebarWidth);
}
sidebarResizer.addEventListener('mousedown', startSidebarDrag);
let orderDir = 'ASC';
const orderDirBtn = document.getElementById('order_dir');
const graphTypeSel = document.getElementById('graph_type');
function updateOrderDirButton() {
orderDirBtn.textContent = orderDir + (orderDir === 'ASC' ? ' \u25B2' : ' \u25B
C');
}
function updateDisplayTypeUI() {
const prevType = displayType;
updateSelectedColumns(prevType);
const newType = graphTypeSel.value;
const showTable = newType === 'table';
const showTS = newType === 'timeseries';
document.getElementById('group_by_field').style.display = showTable || showTS
? 'flex' : 'none';
document.getElementById('aggregate_field').style.display = showTable || showTS
? 'flex' : 'none';
document.getElementById('show_hits_field').style.display = showTable ? 'flex'
: 'none';
document.getElementById('x_axis_field').style.display = showTS ? 'flex' : 'non
e';
...
e.preventDefault();
groupsEl.querySelectorAll('input').forEach(cb => (cb.checked = false));
updateSelectedColumns();
});
updateSelectedColumns();
columnValues.samples = allColumns.slice();
columnValues.table = [];
columnValues.timeseries = [];
groupBy = document.getElementById('group_by').closest('.field');
initChipInput(groupBy, typed =>
allColumns.filter(c => c.toLowerCase().includes(typed.toLowerCase()))
);
initDropdown(orderSelect);
initDropdown(document.getElementById('aggregate'));
});
}
let columnsInitialized = false;
fetch('/api/tables').then(r => r.json()).then(tables => {
tables.forEach(t => {
const o = document.createElement('option');
o.value = t;
o.textContent = t;
document.getElementById('table').appendChild(o);
});
const table = parseSearch().table || tables[0];
document.getElementById('table').value = table;
loadColumns(table).then(() => {
updateDisplayTypeUI();
addFilter();
initFromUrl();
columnsInitialized = true;
});
document.getElementById('table').addEventListener('change', () => {
loadColumns(document.getElementById('table').value).then(() => {
if (columnsInitialized) {
applyParams(parseSearch());
}
});
});
});
document.querySelectorAll('#tabs .tab').forEach(btn => {
btn.addEventListener('click', () => {
document.querySelectorAll('#tabs .tab').forEach(t => t.classList.remove('act
ive'));
document.querySelectorAll('.tab-content').forEach(c => c.classList.remove('a
ctive'));
btn.classList.add('active');
document.getElementById(btn.dataset.tab).classList.add('active');
});
});
document.querySelectorAll('.rel-btn').forEach(btn => {
btn.addEventListener('click', () => {
const dd = document.getElementById(btn.dataset.target);
const show = dd.style.display === 'none' || dd.style.display === '';
document.querySelectorAll('.rel-dropdown').forEach(d => (d.style.display = '
none'));
dd.style.display = show ? 'block' : 'none';
});
});
document.querySelectorAll('.rel-dropdown div').forEach(opt => {
opt.addEventListener('click', () => {
const box = opt.closest('.rel-box');
const input = box.querySelector('input');
input.value = opt.dataset.value || opt.textContent;
opt.parentElement.style.display = 'none';
});
});
document.addEventListener('click', e => {
document.querySelectorAll('.rel-dropdown').forEach(dd => {
if (!dd.parentElement.contains(e.target)) dd.style.display = 'none';
});
});
function updateSelectedColumns(type = graphTypeSel.value) {
const base = allColumns.filter(name => {
const cb = document.querySelector(`#column_groups input[value="${name}"]`);
if (!cb || !cb.checked) return false;
if (type === 'table' && isStringColumn(name)) return false;
return true;
});
if (type === 'table' || type === 'timeseries') {
selectedColumns = groupBy.chips.slice();
if (document.getElementById('show_hits').checked) selectedColumns.push('Hits
');
base.forEach(c => {
if (!selectedColumns.includes(c)) selectedColumns.push(c);
});
derivedColumns.forEach(dc => {
if (dc.include && !selectedColumns.includes(dc.name)) selectedColumns.push
(dc.name);
});
} else {
selectedColumns = base.slice();
derivedColumns.forEach(dc => {
if (dc.include) selectedColumns.push(dc.name);
});
}
columnValues[type] = selectedColumns.slice();
}
function isStringColumn(name) {
const t = (columnTypes[name] || '').toUpperCase();
return t.includes('CHAR') || t.includes('STRING') || t.includes('VARCHAR');
}
function isIntegerColumn(name) {
const t = (columnTypes[name] || '').toUpperCase();
return t.includes('INT');
}
function isTimeColumn(name) {
const t = (columnTypes[name] || '').toUpperCase();
if (t.includes('TIMESTAMP') || t.includes('DATE') || t.includes('TIME')) retur
n true;
const sel = document.getElementById('time_column').value;
const xsel = document.getElementById('x_axis').value;
if (name === sel || name === xsel) return true;
return false;
}
function formatNumber(val) {I'm continuing to scroll, getting closer to the 800 mark now.
x.html
function formatNumber(val) {
if (typeof val !== 'number') val = Number(val);
if (Number.isNaN(val)) return '';
if (val === 0) return '0';
const abs = Math.abs(val);
if (abs > 999.999) {
const units = [
{n: 1e12, s: 'T'},
{n: 1e9, s: 'B'},
{n: 1e6, s: 'M'},
{n: 1e3, s: 'K'},
];
for (const u of units) {
if (abs >= u.n) {
return (val / u.n).toFixed(2) + ' ' + u.s;
}
}
}
if (abs < 0.0005) return '0.000';
return val.toFixed(3);
}
function addFilter() {
const container = document.createElement('div');
container.className = 'filter';
container.innerHTML = `
<div class="filter-row">
<select class="f-col"></select>
<select class="f-op"></select>
<button type="button" class="remove" onclick="this.closest('.filter').remo
ve()">✖</button>
</div>
<div class="chip-box">
<div class="chip-input">
<input class="f-val" type="text">
<button type="button" class="chip-copy">⎘</button>
</div>
<div class="chip-dropdown"></div>
</div>
`;
const colSel = container.querySelector('.f-col');
colSel.innerHTML = allColumns.map(c => `<option value="${c}">${c}</option>`).j
oin('');
initDropdown(colSel);
function populateOps() {
const opSel = container.querySelector('.f-op');
const col = colSel.value;
const ops = isStringColumn(col)
? [
['=', '='],
['!=', '!='],
['~', 'matches regex'],
['!~', 'not matches regex'],
['contains', 'contains'],
['!contains', 'not contains'],
['empty', 'empty'],
['!empty', 'not empty'],
['LIKE', 'like'],
]
: [
['=', '='],
['!=', '!='],
['<', '<'],
['>', '>'],
];
opSel.innerHTML = ops.map(o => `<option value="${o[0]}">${o[1]}</option>`).j
oin('');
updateInputVis();
}
function updateInputVis() {
const op = container.querySelector('.f-op').value;
const box = container.querySelector('.chip-box');
box.style.display = op === 'empty' || op === '!empty' ? 'none' : 'block';
}
colSel.addEventListener('change', populateOps);
container.querySelector('.f-op').addEventListener('change', updateInputVis);
populateOps();
document.getElementById('filter_list').appendChild(container);
initChipInput(container, (typed, el) => {
const colEl = el.querySelector('.f-col select') || el.querySelector('.f-col'
);
if (!colEl) return [];
const col = colEl.value;
if (!isStringColumn(col)) return [];
return fetch(`/api/samples?column=${encodeURIComponent(col)}&q=${encodeURICo
mponent(typed)}`)
.then(r => r.json());
});
}
function nextDerivedName() {
let n = 1;
while (true) {
const name = `derived_${n}`;
if (!derivedColumns.some(d => d.name === name) && !allColumns.includes(name)
) return name;
n++;
}
}
function addDerived(data = {}) {
const container = document.createElement('div');
container.className = 'derived';
container.innerHTML = `
<div class="derived-row">
<select class="d-type">
<option value="aggregated">Aggregated</option>
<option value="string">String</option>
<option value="numeric">Numeric</option>
</select>
<input class="d-name" type="text">
<button type="button" class="remove" onclick="removeDerived(this)">✖</butt
on>
</div>
<label><input type="checkbox" class="d-use" checked> Include in Query</label
>
<textarea class="d-expr" rows="2"></textarea>
`;
document.getElementById('derived_list').appendChild(container);
...
orderDir = params.order_dir || 'ASC';
updateOrderDirButton();
if (params.limit !== undefined && params.limit !== null) {
document.getElementById('limit').value = params.limit;
limitValues[params.graph_type || 'samples'] = params.limit;
limitInput.dataset.setByUser = '1';
}
graphTypeSel.value = params.graph_type || 'samples';
updateDisplayTypeUI();
limitInput.value = limitValues[graphTypeSel.value];
if (params.x_axis) {
document.getElementById('x_axis').value = params.x_axis;
} else {
document.getElementById('x_axis').value = '';
}
if (params.granularity) document.getElementById('granularity').value = params.
granularity;
if (params.fill) document.getElementById('fill').value = params.fill;
if (params.group_by) {
groupBy.chips.splice(0, groupBy.chips.length, ...params.group_by);
groupBy.renderChips();
}
if (params.aggregate) document.getElementById('aggregate').value = params.aggr
egate;
document.getElementById('show_hits').checked = params.show_hits ?? true;
if (params.samples_columns) columnValues.samples = params.samples_columns;
if (params.table_columns) columnValues.table = params.table_columns;
if (params.timeseries_columns) columnValues.timeseries = params.timeseries_col
umns;
document.querySelectorAll('#column_groups input').forEach(cb => {
cb.checked = columnValues[graphTypeSel.value].includes(cb.value);
});
updateSelectedColumns(graphTypeSel.value);
const dlist = document.getElementById('derived_list');
dlist.innerHTML = '';
derivedColumns.splice(0, derivedColumns.length);
if (params.derived_columns && params.derived_columns.length) {
params.derived_columns.forEach(dc => addDerived(dc));
}
refreshDerivedColumns();
const list = document.getElementById('filter_list');
list.innerHTML = '';
if (params.filters && params.filters.length) {
params.filters.forEach(f => {
addFilter();
const el = list.lastElementChild;
const colSel = el.querySelector('.f-col select') || el.querySelector('.f-c
ol');
colSel.value = f.column;
colSel.dispatchEvent(new Event('change'));
el.querySelector('.f-op').value = f.op;
el.querySelector('.f-op').dispatchEvent(new Event('change'));
if (f.value !== null && f.op !== 'empty' && f.op !== '!empty') {
const values = Array.isArray(f.value) ? f.value : [f.value];
values.forEach(v => el.addChip(v));
el.renderChips();
}
});
} else {
addFilter();
}
}
function parseSearch() {
const sp = new URLSearchParams(window.location.search);
const params = {};
if (sp.has('table')) params.table = sp.get('table');
if (sp.has('time_column')) params.time_column = sp.get('time_column');
if (sp.has('time_unit')) params.time_unit = sp.get('time_unit');
if (sp.has('start')) params.start = sp.get('start');
if (sp.has('end')) params.end = sp.get('end');
if (sp.has('order_by')) params.order_by = sp.get('order_by');
if (sp.has('order_dir')) params.order_dir = sp.get('order_dir');
if (sp.has('limit')) params.limit = parseInt(sp.get('limit'), 10);
if (sp.has('samples_columns')) params.samples_columns = sp.get('samples_column
s').split(',').filter(c => c);
if (sp.has('table_columns')) params.table_columns = sp.get('table_columns').sp
lit(',').filter(c => c);
if (sp.has('timeseries_columns')) params.timeseries_columns = sp.get('timeseri
es_columns').split(',').filter(c => c);
if (sp.has('filters')) {
try { params.filters = JSON.parse(sp.get('filters')); } catch(e) { params.fi
lters = []; }
}
if (sp.has('graph_type')) params.graph_type = sp.get('graph_type');
if (sp.has('group_by')) params.group_by = sp.get('group_by').split(',').filter
(c => c);
if (sp.has('aggregate')) params.aggregate = sp.get('aggregate');
if (sp.has('show_hits')) params.show_hits = sp.get('show_hits') === '1';
if (sp.has('x_axis')) params.x_axis = sp.get('x_axis');
if (sp.has('granularity')) params.granularity = sp.get('granularity');
if (sp.has('fill')) params.fill = sp.get('fill');
if (sp.has('derived_columns')) {
try { params.derived_columns = JSON.parse(sp.get('derived_columns')); } catc
h(e) { params.derived_columns = []; }
}
return params;
}
function initFromUrl() {
const params = parseSearch();
history.replaceState(params, '', paramsToSearch(params));
applyParams(params);
dive(false);
}
window.addEventListener('popstate', e => {
const params = e.state || parseSearch();
applyParams(params);
dive(false);
});
let originalRows = [];
let sortState = {index: null, dir: null};
function renderTable(rows) {
const table = document.getElementById('results');
table.innerHTML = '';
if (rows.length === 0) return;
let hitsIndex = selectedColumns.indexOf('Hits');
let totalHits = 0;
if (hitsIndex !== -1) {
totalHits = rows.reduce((s, r) => s + Number(r[hitsIndex]), 0);
}I’m looking up loadColumns in JS to check it out.
root@ee1d42b3cdab:/workspace/scubaduck# grep -n "loadColumns" -n scubaduck/stati
c/index.html
549:function loadColumns(table) {
710: loadColumns(table).then(() => {
717: loadColumns(document.getElementById('table').value).then(() => {root@ee1d42b3cdab:/workspace/scubaduck# sed -n '548,620p' scubaduck/static/index
.html
function loadColumns(table) {
return fetch('/api/columns?table=' + encodeURIComponent(table)).then(r => r.js
on()).then(cols => {
const orderSelect = document.getElementById('order_by');
const xAxisSelect = document.getElementById('x_axis');
const groupsEl = document.getElementById('column_groups');
const timeColumnSelect = document.getElementById('time_column');
orderSelect.innerHTML = '';
xAxisSelect.innerHTML = '';
const defOpt = document.createElement('option');
defOpt.value = '';
defOpt.textContent = '(default)';
xAxisSelect.appendChild(defOpt);
timeColumnSelect.innerHTML = '';
groupsEl.innerHTML = '';
allColumns.length = 0;
stringColumns.length = 0;
integerColumns.length = 0;
timeColumns.length = 0;
timeColumnOptions.length = 0;
baseColumns.length = 0;
baseStringColumns.length = 0;
baseIntegerColumns.length = 0;
baseTimeColumns.length = 0;
baseTimeColumnOptions.length = 0;
for (const k in columnTypes) delete columnTypes[k];
const groups = {
time: {name: 'Time', cols: [], ul: null},
integer: {name: 'Integers', cols: [], ul: null},
string: {name: 'Strings', cols: [], ul: null},
};
const heur = ['timestamp','created','created_at','event_time','time','date',
'occurred','happened','logged'];
let guess = null;
cols.forEach(c => {
const t = c.type.toUpperCase();
columnTypes[c.name] = c.type;
allColumns.push(c.name);
baseColumns.push(c.name);
let g = 'string';
const isNumeric = t.includes('INT') || t.includes('DECIMAL') || t.includes
('NUMERIC') || t.includes('REAL') || t.includes('DOUBLE') || t.includes('FLOAT')
|| t.includes('HUGEINT');
const isTimeType = t.includes('TIMESTAMP') || t.includes('DATE') || t.incl
udes('TIME');
if (isNumeric || isTimeType) {
timeColumnOptions.push(c.name);
baseTimeColumnOptions.push(c.name);
if (!guess && heur.some(h => c.name.toLowerCase().includes(h))) guess =
c.name;
}
if (isTimeType) {
g = 'time';
timeColumns.push(c.name);
baseTimeColumns.push(c.name);
} else if (isNumeric) {
g = 'integer';
}
if (g === 'string') {
stringColumns.push(c.name);
baseStringColumns.push(c.name);
} else if (g === 'integer') {
integerColumns.push(c.name);
baseIntegerColumns.push(c.name);
}
groups[g].cols.push(c.name);
if (g !== 'string') {
const o = document.createElement('option');
o.value = c.name;
o.textContent = c.name;
orderSelect.appendChild(o);
}
});
timeColumns.forEach(name => {
const o = document.createElement('option');
o.value = name;
o.textContent = name;
xAxisSelect.appendChild(o);root@ee1d42b3cdab:/workspace/scubaduck# sed -n '620,680p' scubaduck/static/index
.html
xAxisSelect.appendChild(o);
});
timeColumnOptions.forEach(name => {
const o = document.createElement('option');
o.value = name;
o.textContent = name;
timeColumnSelect.appendChild(o);
});
xAxisSelect.value = '';
defaultTimeColumn = guess || timeColumnOptions[0] || '';
Object.keys(groups).forEach(key => {
const g = groups[key];
const div = document.createElement('div');
div.className = 'col-group';
const header = document.createElement('div');
header.className = 'col-group-header';
header.appendChild(document.createTextNode(g.name + ': '));
const links = document.createElement('span');
links.className = 'links';
const allBtn = document.createElement('a');
allBtn.href = '#';
allBtn.textContent = 'All';
const noneBtn = document.createElement('a');
noneBtn.href = '#';
noneBtn.textContent = 'None';
links.appendChild(allBtn);
links.appendChild(noneBtn);
header.appendChild(links);
div.appendChild(header);
const ul = document.createElement('ul');
g.ul = ul;
g.cols.forEach(name => {
const li = document.createElement('li');
const label = document.createElement('label');
const cb = document.createElement('input');
cb.type = 'checkbox';
cb.value = name;
cb.checked = true;
cb.addEventListener('change', updateSelectedColumns);
label.appendChild(cb);
label.appendChild(document.createTextNode(' ' + name));
li.appendChild(label);
ul.appendChild(li);
});
allBtn.addEventListener('click', e => {
e.preventDefault();
ul.querySelectorAll('input').forEach(cb => (cb.checked = true));
updateSelectedColumns();
});
noneBtn.addEventListener('click', e => {
e.preventDefault();
ul.querySelectorAll('input').forEach(cb => (cb.checked = false));
updateSelectedColumns();
});
div.appendChild(ul);
groupsEl.appendChild(div);
});
document.getElementById('columns_all').addEventListener('click', e => {
e.preventDefault();
groupsEl.querySelectorAll('input').forEach(cb => (cb.checked = true));
updateSelectedColumns();root@ee1d42b3cdab:/workspace/scubaduck# sed -n '680,760p' scubaduck/static/index
.html
updateSelectedColumns();
});
document.getElementById('columns_none').addEventListener('click', e => {
e.preventDefault();
groupsEl.querySelectorAll('input').forEach(cb => (cb.checked = false));
updateSelectedColumns();
});
updateSelectedColumns();
columnValues.samples = allColumns.slice();
columnValues.table = [];
columnValues.timeseries = [];
groupBy = document.getElementById('group_by').closest('.field');
initChipInput(groupBy, typed =>
allColumns.filter(c => c.toLowerCase().includes(typed.toLowerCase()))
);
initDropdown(orderSelect);
initDropdown(document.getElementById('aggregate'));
});
}
let columnsInitialized = false;
fetch('/api/tables').then(r => r.json()).then(tables => {
tables.forEach(t => {
const o = document.createElement('option');
o.value = t;
o.textContent = t;
document.getElementById('table').appendChild(o);
});
const table = parseSearch().table || tables[0];
document.getElementById('table').value = table;
loadColumns(table).then(() => {
updateDisplayTypeUI();
addFilter();
initFromUrl();
columnsInitialized = true;
});
document.getElementById('table').addEventListener('change', () => {
loadColumns(document.getElementById('table').value).then(() => {
if (columnsInitialized) {
applyParams(parseSearch());
}
});
});
});
document.querySelectorAll('#tabs .tab').forEach(btn => {
btn.addEventListener('click', () => {
document.querySelectorAll('#tabs .tab').forEach(t => t.classList.remove('act
ive'));
document.querySelectorAll('.tab-content').forEach(c => c.classList.remove('a
ctive'));
btn.classList.add('active');
document.getElementById(btn.dataset.tab).classList.add('active');
});
});
document.querySelectorAll('.rel-btn').forEach(btn => {
btn.addEventListener('click', () => {
const dd = document.getElementById(btn.dataset.target);
const show = dd.style.display === 'none' || dd.style.display === '';
document.querySelectorAll('.rel-dropdown').forEach(d => (d.style.display = '
none'));
dd.style.display = show ? 'block' : 'none';
});
});
document.querySelectorAll('.rel-dropdown div').forEach(opt => {
opt.addEventListener('click', () => {
const box = opt.closest('.rel-box');
const input = box.querySelector('input');
input.value = opt.dataset.value || opt.textContent;
opt.parentElement.style.display = 'none';
});
});
document.addEventListener('click', e => {
document.querySelectorAll('.rel-dropdown').forEach(dd => {
if (!dd.parentElement.contains(e.target)) dd.style.display = 'none';
});
});
function updateSelectedColumns(type = graphTypeSel.value) {
const base = allColumns.filter(name => {
const cb = document.querySelector(`#column_groups input[value="${name}"]`);
if (!cb || !cb.checked) return false;
if (type === 'table' && isStringColumn(name)) return false;I'm scrolling through lines 760 to 1040 to check the updateTimeColumn function.
root@ee1d42b3cdab:/workspace/scubaduck# grep -n "time_column" -n -n scubaduck/st
atic/index.html
203: <select id="time_column"></select>
554: const timeColumnSelect = document.getElementById('time_column');
794: const sel = document.getElementById('time_column').value;
1003: time_column: document.getElementById('time_column').value,
1051: if (params.time_column) sp.set('time_column', params.time_column);
1080: document.getElementById('time_column').value = params.time_column || defa
ultTimeColumn;
1150: if (sp.has('time_column')) params.time_column = sp.get('time_column');I'll check around line 780 to review the code for isTimeColumn.
x.html
function isStringColumn(name) {
const t = (columnTypes[name] || '').toUpperCase();
return t.includes('CHAR') || t.includes('STRING') || t.includes('VARCHAR');
}
function isIntegerColumn(name) {
const t = (columnTypes[name] || '').toUpperCase();
return t.includes('INT');
}
function isTimeColumn(name) {
const t = (columnTypes[name] || '').toUpperCase();
if (t.includes('TIMESTAMP') || t.includes('DATE') || t.includes('TIME')) retur
n true;
const sel = document.getElementById('time_column').value;
const xsel = document.getElementById('x_axis').value;
if (name === sel || name === xsel) return true;
return false;
}
function formatNumber(val) {
if (typeof val !== 'number') val = Number(val);
if (Number.isNaN(val)) return '';
if (val === 0) return '0';
const abs = Math.abs(val);
if (abs > 999.999) {
const units = [
{n: 1e12, s: 'T'},
{n: 1e9, s: 'B'},
{n: 1e6, s: 'M'},
{n: 1e3, s: 'K'},
];
for (const u of units) {
if (abs >= u.n) {
return (val / u.n).toFixed(2) + ' ' + u.s;
}
}
}
if (abs < 0.0005) return '0.000';
return val.toFixed(3);
}
function addFilter() {
const container = document.createElement('div');
container.className = 'filter';
container.innerHTML = `
<div class="filter-row">
<select class="f-col"></select>
<select class="f-op"></select>
<button type="button" class="remove" onclick="this.closest('.filter').remo
ve()">✖</button>
</div>
<div class="chip-box">
<div class="chip-input">
<input class="f-val" type="text">
<button type="button" class="chip-copy">⎘</button>
</div>
<div class="chip-dropdown"></div>
</div>
`;
const colSel = container.querySelector('.f-col');
colSel.innerHTML = allColumns.map(c => `<option value="${c}">${c}</option>`).j
oin('');
initDropdown(colSel);
function populateOps() {
const opSel = container.querySelector('.f-op');
const col = colSel.value;
const ops = isStringColumn(col)
? [
['=', '='],
['!=', '!='],
['~', 'matches regex'],
['!~', 'not matches regex'],
['contains', 'contains'],
['!contains', 'not contains'],
['empty', 'empty'],
['!empty', 'not empty'],
['LIKE', 'like'],
]
: [
['=', '='],
['!=', '!='],
['<', '<'],
['>', '>'],
];
opSel.innerHTML = ops.map(o => `<option value="${o[0]}">${o[1]}</option>`).j
oin('');
updateInputVis();
}
function updateInputVis() {
const op = container.querySelector('.f-op').value;
const box = container.querySelector('.chip-box');
box.style.display = op === 'empty' || op === '!empty' ? 'none' : 'block';
}
colSel.addEventListener('change', populateOps);
container.querySelector('.f-op').addEventListener('change', updateInputVis);
populateOps();
document.getElementById('filter_list').appendChild(container);
initChipInput(container, (typed, el) => {
const colEl = el.querySelector('.f-col select') || el.querySelector('.f-col'
);
if (!colEl) return [];
const col = colEl.value;
if (!isStringColumn(col)) return [];
return fetch(`/api/samples?column=${encodeURIComponent(col)}&q=${encodeURICo
mponent(typed)}`)
.then(r => r.json());
});
}
function nextDerivedName() {
let n = 1;
while (true) {
const name = `derived_${n}`;
if (!derivedColumns.some(d => d.name === name) && !allColumns.includes(name)
) return name;
n++;
}
...
container.querySelector('.d-name').value = obj.name;
container.querySelector('.d-expr').value = obj.expr;
container.querySelector('.d-use').checked = obj.include;
['change','input'].forEach(evt => {
container.addEventListener(evt, refreshDerivedColumns);
});
derivedColumns.push(obj);
refreshDerivedColumns();
}
function removeDerived(btn) {
const el = btn.closest('.derived');
const idx = derivedColumns.findIndex(d => d.el === el);
if (idx !== -1) {
derivedColumns.splice(idx, 1);
}
el.remove();
refreshDerivedColumns();
}
function refreshDerivedColumns() {
allColumns.splice(0, allColumns.length, ...baseColumns);
stringColumns.splice(0, stringColumns.length, ...baseStringColumns);
integerColumns.splice(0, integerColumns.length, ...baseIntegerColumns);
timeColumns.splice(0, timeColumns.length, ...baseTimeColumns);
timeColumnOptions.splice(0, timeColumnOptions.length, ...baseTimeColumnOptions
);
derivedColumns.forEach(d => {
d.type = d.el.querySelector('.d-type').value;
d.name = d.el.querySelector('.d-name').value;
d.expr = d.el.querySelector('.d-expr').value;
d.include = d.el.querySelector('.d-use').checked;
allColumns.push(d.name);
columnTypes[d.name] = d.type;
if (d.type === 'string') {
stringColumns.push(d.name);
} else {
integerColumns.push(d.name);
timeColumnOptions.push(d.name);
}
});
updateSelectedColumns();
}
let lastQueryTime = 0;
let queryStart = 0;
function dive(push=true) {
const params = collectParams();
if (push) {
history.pushState(params, '', paramsToSearch(params));
}
const payload = Object.assign({}, params);
const dcMap = {};
(params.derived_columns || []).forEach(d => {
if (d.include) dcMap[d.name] = d.expr;
});
payload.derived_columns = dcMap;
const view = document.getElementById('view');
view.innerHTML = '<p>Loading...</p>';
window.lastResults = undefined;
queryStart = performance.now();
fetch('/api/query', {method:'POST', headers:{'Content-Type':'application/json'
}, body:JSON.stringify(payload)})
.then(async r => {
const data = await r.json();
if (!r.ok) throw data;
return data;
})
.then(data => {
lastQueryTime = Math.round(performance.now() - queryStart);
showResults(data);
})
.catch(err => {
showError(err);
});
}
function collectParams() {
updateSelectedColumns();
const payload = {
table: document.getElementById('table').value,
time_column: document.getElementById('time_column').value,
time_unit: document.getElementById('time_unit').value,
start: document.getElementById('start').value,
end: document.getElementById('end').value,
order_by: document.getElementById('order_by').value,
order_dir: orderDir,
limit: parseInt(document.getElementById('limit').value, 10),
columns: selectedColumns.filter(c =>
c !== 'Hits' && !derivedColumns.some(dc => dc.name === c)
),
samples_columns: columnValues.samples.slice(),
table_columns: columnValues.table.slice(),
timeseries_columns: columnValues.timeseries.slice(),
graph_type: graphTypeSel.value,
filters: Array.from(document.querySelectorAll('#filters .filter')).map(f =>
{
const chips = f.chips || [];
const op = f.querySelector('.f-op').value;
let value = null;
if (op !== 'empty' && op !== '!empty') {
value = chips.length === 0 ? null : (chips.length === 1 ? chips[0] : chi
ps);
}
const colSel = f.querySelector('.f-col select') || f.querySelector('.f-col
');
return {column: colSel.value, op, value};
}),
derived_columns: Array.from(document.querySelectorAll('#derived_list .derive
d')).map(d => ({
type: d.querySelector('.d-type').value,
name: d.querySelector('.d-name').value,
expr: d.querySelector('.d-expr').value,
include: d.querySelector('.d-use').checked,
}))
};
if (graphTypeSel.value === 'table' || graphTypeSel.value === 'timeseries') {
payload.group_by = groupBy.chips || [];
payload.aggregate = document.getElementById('aggregate').value;
payload.show_hits = document.getElementById('show_hits').checked;
}
if (graphTypeSel.value === 'timeseries') {
const xval = document.getElementById('x_axis').value;root@ee1d42b3cdab:/workspace/scubaduck# sed -n '1040,1320p' scubaduck/static/ind
ex.html
const xval = document.getElementById('x_axis').value;
if (xval) payload.x_axis = xval;
payload.granularity = document.getElementById('granularity').value;
payload.fill = document.getElementById('fill').value;
}
return payload;
}
function paramsToSearch(params) {
const sp = new URLSearchParams();
if (params.table) sp.set('table', params.table);
if (params.time_column) sp.set('time_column', params.time_column);
if (params.time_unit) sp.set('time_unit', params.time_unit);
if (params.start) sp.set('start', params.start);
if (params.end) sp.set('end', params.end);
if (params.order_by) sp.set('order_by', params.order_by);
if (params.order_dir) sp.set('order_dir', params.order_dir);
if (params.limit !== null && params.limit !== undefined) sp.set('limit', param
s.limit);
if (params.samples_columns && params.samples_columns.length) sp.set('samples_c
olumns', params.samples_columns.join(','));
if (params.table_columns && params.table_columns.length) sp.set('table_columns
', params.table_columns.join(','));
if (params.timeseries_columns && params.timeseries_columns.length) sp.set('tim
eseries_columns', params.timeseries_columns.join(','));
if (params.filters && params.filters.length) sp.set('filters', JSON.stringify(
params.filters));
if (params.derived_columns && params.derived_columns.length) sp.set('derived_c
olumns', JSON.stringify(params.derived_columns));
if (params.graph_type) sp.set('graph_type', params.graph_type);
if (params.graph_type === 'table' || params.graph_type === 'timeseries') {
if (params.group_by && params.group_by.length) sp.set('group_by', params.gro
up_by.join(','));
if (params.aggregate) sp.set('aggregate', params.aggregate);
if (params.show_hits) sp.set('show_hits', '1');
}
if (params.graph_type === 'timeseries') {
if (params.x_axis) sp.set('x_axis', params.x_axis);
if (params.granularity) sp.set('granularity', params.granularity);
if (params.fill) sp.set('fill', params.fill);
}
const qs = sp.toString();
return qs ? '?' + qs : '';
}
function applyParams(params) {
if (params.table) document.getElementById('table').value = params.table;
document.getElementById('time_column').value = params.time_column || defaultTi
meColumn;
if (params.time_unit) document.getElementById('time_unit').value = params.time
_unit;
document.getElementById('start').value = params.start || '';
document.getElementById('end').value = params.end || '';
if (params.order_by) {
document.getElementById('order_by').value = params.order_by;
}
orderDir = params.order_dir || 'ASC';
updateOrderDirButton();
if (params.limit !== undefined && params.limit !== null) {
document.getElementById('limit').value = params.limit;
limitValues[params.graph_type || 'samples'] = params.limit;
limitInput.dataset.setByUser = '1';
}
graphTypeSel.value = params.graph_type || 'samples';
updateDisplayTypeUI();
limitInput.value = limitValues[graphTypeSel.value];
if (params.x_axis) {
document.getElementById('x_axis').value = params.x_axis;
} else {
document.getElementById('x_axis').value = '';
}
if (params.granularity) document.getElementById('granularity').value = params.
granularity;
if (params.fill) document.getElementById('fill').value = params.fill;
if (params.group_by) {
groupBy.chips.splice(0, groupBy.chips.length, ...params.group_by);
groupBy.renderChips();
}
if (params.aggregate) document.getElementById('aggregate').value = params.aggr
egate;
document.getElementById('show_hits').checked = params.show_hits ?? true;
if (params.samples_columns) columnValues.samples = params.samples_columns;
if (params.table_columns) columnValues.table = params.table_columns;
if (params.timeseries_columns) columnValues.timeseries = params.timeseries_col
umns;
document.querySelectorAll('#column_groups input').forEach(cb => {
cb.checked = columnValues[graphTypeSel.value].includes(cb.value);
});
updateSelectedColumns(graphTypeSel.value);
const dlist = document.getElementById('derived_list');
dlist.innerHTML = '';
derivedColumns.splice(0, derivedColumns.length);
if (params.derived_columns && params.derived_columns.length) {
params.derived_columns.forEach(dc => addDerived(dc));
}
refreshDerivedColumns();
const list = document.getElementById('filter_list');
list.innerHTML = '';
if (params.filters && params.filters.length) {
params.filters.forEach(f => {
addFilter();
const el = list.lastElementChild;
const colSel = el.querySelector('.f-col select') || el.querySelector('.f-c
ol');
colSel.value = f.column;
colSel.dispatchEvent(new Event('change'));
el.querySelector('.f-op').value = f.op;
el.querySelector('.f-op').dispatchEvent(new Event('change'));
if (f.value !== null && f.op !== 'empty' && f.op !== '!empty') {
const values = Array.isArray(f.value) ? f.value : [f.value];
values.forEach(v => el.addChip(v));
el.renderChips();
}
});
} else {
addFilter();
}
}
function parseSearch() {
const sp = new URLSearchParams(window.location.search);
const params = {};
...
totalHits = rows.reduce((s, r) => s + Number(r[hitsIndex]), 0);
}
const header = document.createElement('tr');
selectedColumns.forEach((col, i) => {
const th = document.createElement('th');
let label = col;
if (
displayType === 'table' &&
col !== 'Hits' &&
!(groupBy.chips || []).includes(col)
) {
const agg = document.getElementById('aggregate').value.toLowerCase();
label += ` (${agg})`;
}
th.textContent = label;
th.dataset.index = i;
th.addEventListener('click', handleSort);
if (sortState.index === i) {
th.classList.add('sorted');
th.textContent =
label + (sortState.dir === 'desc' ? ' \u25BC' : ' \u25B2');
}
if (!isStringColumn(col)) th.style.textAlign = 'right';
header.appendChild(th);
});
table.appendChild(header);
rows.forEach(row => {
const tr = document.createElement('tr');
tr.addEventListener('click', () => {
const wasSelected = tr.classList.contains('selected');
document
.querySelectorAll('#results tr.selected')
.forEach(el => el.classList.remove('selected'));
if (!wasSelected) {
tr.classList.add('selected');
}
});
row.forEach((v, i) => {
const col = selectedColumns[i];
const td = document.createElement('td');
if (isTimeColumn(col)) {
let d;
const t = (columnTypes[col] || '').toUpperCase();
if (t.includes('TIMESTAMP') || t.includes('DATE') || t.includes('TIME'))
{
d = new Date(v);
} else {
const unit = document.getElementById('time_unit').value;
const factors = {s: 1000, ms: 1, us: 0.001, ns: 0.000001};
d = new Date(Number(v) * (factors[unit] || 1000));
}
td.textContent = d.toLocaleString('en-US', {
weekday: 'short',
month: 'short',
day: 'numeric',
year: 'numeric',
hour: 'numeric',
minute: 'numeric',
second: 'numeric',
hour12: true,
timeZoneName: 'short'
});
} else {
if (col === 'Hits') {
const pct = totalHits ? ((v / totalHits) * 100).toFixed(1) : '0';
td.textContent = `${formatNumber(v)} (${pct}%)`;
} else {
td.textContent = isStringColumn(col) ? v : formatNumber(v);
}
}
if (!isStringColumn(col) && !isTimeColumn(col)) {
td.classList.add('numeric');
}
td.style.textAlign = isStringColumn(col) ? 'left' : 'right';
tr.appendChild(td);
});
table.appendChild(tr);
});
// ensure table does not overflow unless necessary
const view = document.getElementById('view');
if (table.scrollWidth <= view.clientWidth) {
table.style.width = '100%';
}
}
function handleSort(e) {
const idx = parseInt(e.target.dataset.index, 10);
if (sortState.index !== idx) {
sortState.index = idx;
sortState.dir = 'asc';
} else if (sortState.dir === 'asc') {
sortState.dir = 'desc';
} else if (sortState.dir === 'desc') {
sortState.index = null;
sortState.dir = null;
} else {
sortState.dir = 'asc';
}
let rows = originalRows.slice();
if (sortState.index !== null) {
rows.sort((a, b) => {
const va = a[sortState.index];
const vb = b[sortState.index];
if (va === vb) return 0;
if (sortState.dir === 'desc') return va < vb ? 1 : -1;
return va > vb ? 1 : -1;
});
}
renderTable(rows);
}
function showResults(data) {
window.lastResults = data;
const view = document.getElementById('view');
if (graphTypeSel.value === 'timeseries') {
showTimeSeries(data);
} else {
if (data.rows.length === 0) {
view.innerHTML =
'<p id="empty-message">Empty data provided to table</p><table id="result
s"></table>';
} else {
view.innerHTML = '<table id="results"></table>';ex.html
view.innerHTML = '<table id="results"></table>';
}
originalRows = data.rows.slice();
sortState = {index: null, dir: null};
renderTable(originalRows);
}
const sqlEl = document.createElement('pre');
sqlEl.id = 'sql_query';
sqlEl.style.whiteSpace = 'pre-wrap';
sqlEl.style.marginTop = '10px';
sqlEl.textContent = data.sql;
view.appendChild(sqlEl);
document.getElementById('query_info').textContent =
`Your query took about ${lastQueryTime} ms`;
}
function showError(err) {
window.lastResults = err;
const view = document.getElementById('view');
let msg = '';
if (typeof err === 'string') {
msg = err;
} else if (err) {
msg = err.error || 'Error';
if (err.sql) {
msg += '\nSQL: ' + err.sql;
}
if (err.traceback) {
msg += '\n' + err.traceback;
}
}
view.innerHTML = `<pre id="error-message">${msg}</pre>`;
document.getElementById('query_info').textContent = '';
}
function setSelectValue(selector, value) {
const el = typeof selector === 'string' ? document.querySelector(selector) : s
elector;
if (el) {
const select = el.tagName === 'SELECT' ? el : el.querySelector('select');
if (select) {
select.value = value;
select.dispatchEvent(new Event('change'));
}
}
}
</script>
</body>
</html>I’m searching for where defaultTimeColumn is used in the code.
root@ee1d42b3cdab:/workspace/scubaduck# grep -n "defaultTimeColumn" -n scubaduck
/static/index.html
370:let defaultTimeColumn = '';
629: defaultTimeColumn = guess || timeColumnOptions[0] || '';
1080: document.getElementById('time_column').value = params.time_column || defa
ultTimeColumn;I'm looking for timeColumns.length and considering adding a "(none)" option for no time.
root@ee1d42b3cdab:/workspace/scubaduck# grep -n "start" -n scubaduck/static/inde
x.html | head
212: <label>Start<span class="help" title="Sets the start/end of the ti
me range to query. Can be any kind of datetime string. For example: 'April 23, 2
014' or 'yesterday'.">[?]</span></label>
214: <input id="start" type="text" />
215: <button type="button" class="rel-btn" data-target="start-select"
>▼</button>
216: <div id="start-select" class="rel-dropdown" data-input="start">
230: <label>End<span class="help" title="Sets the start/end of the time
range to query. Can be any kind of datetime string. For example: 'April 23, 201
4' or 'yesterday'.">[?]</span></label>
481:function startSidebarDrag(e) {
498:sidebarResizer.addEventListener('mousedown', startSidebarDrag);
519: if (g.querySelector('.col-group-header').textContent.startsWith('Strings
')) {
1005: start: document.getElementById('start').value,
1053: if (params.start) sp.set('start', params.start);root@ee1d42b3cdab:/workspace/scubaduck# grep -n "displayTypeUI" -n scubaduck/sta
tic/index.htmlI'll look for the 'display:none;' element; it might be toggled by JavaScript.
root@ee1d42b3cdab:/workspace/scubaduck# grep -n "Start'" -n scubaduck/static/ind
ex.htmlI’m wondering if there’s a function like updateTimeColumn to help with this.
root@ee1d42b3cdab:/workspace/scubaduck# grep -n "time_column" -n scubaduck/stati
c/index.html | head -n 20
203: <select id="time_column"></select>
554: const timeColumnSelect = document.getElementById('time_column');
794: const sel = document.getElementById('time_column').value;
1003: time_column: document.getElementById('time_column').value,
1051: if (params.time_column) sp.set('time_column', params.time_column);
1080: document.getElementById('time_column').value = params.time_column || defa
ultTimeColumn;
1150: if (sp.has('time_column')) params.time_column = sp.get('time_column');I'll take a closer look at the code around line 790.
root@ee1d42b3cdab:/workspace/scubaduck# sed -n '780,820p' scubaduck/static/index
.html
function isStringColumn(name) {
const t = (columnTypes[name] || '').toUpperCase();
return t.includes('CHAR') || t.includes('STRING') || t.includes('VARCHAR');
}
function isIntegerColumn(name) {
const t = (columnTypes[name] || '').toUpperCase();
return t.includes('INT');
}
function isTimeColumn(name) {
const t = (columnTypes[name] || '').toUpperCase();
if (t.includes('TIMESTAMP') || t.includes('DATE') || t.includes('TIME')) retur
n true;
const sel = document.getElementById('time_column').value;
const xsel = document.getElementById('x_axis').value;
if (name === sel || name === xsel) return true;
return false;
}
function formatNumber(val) {
if (typeof val !== 'number') val = Number(val);
if (Number.isNaN(val)) return '';
if (val === 0) return '0';
const abs = Math.abs(val);
if (abs > 999.999) {
const units = [
{n: 1e12, s: 'T'},
{n: 1e9, s: 'B'},
{n: 1e6, s: 'M'},
{n: 1e3, s: 'K'},
];
for (const u of units) {
if (abs >= u.n) {
return (val / u.n).toFixed(2) + ' ' + u.s;
}
}
}
if (abs < 0.0005) return '0.000';
return val.toFixed(3);
}I see collectParams and paramsToSearch use time_column, but no explicit function for Start/End visibility.
root@ee1d42b3cdab:/workspace/scubaduck# grep -n "time_column" -n scubaduck/stati
c/js -rI'm thinking of adjusting the server's query to handle a "(none)" time column.
I’ll modify the query to accept empty or None time_column, without breaking existing tests.
I’ll modify checks to gracefully handle a None time_column for start/end computation.
I'll skip start/end filters if time_column is "(none)," but allow x_axis selection. Testing required.
root@ee1d42b3cdab:/workspace/scubaduck# grep -n "default_start_end_returned" -n
tests/test_server.py
775:def test_default_start_end_returned() -> None:assert data["rows"][0][1] == 20
def test_timeseries_derived_column() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "timeseries",
"granularity": "1 hour",
"limit": 7,
"columns": ["value"],
"derived_columns": {"derived_1": "value * 2"},
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
rows = data["rows"]
assert all(r[2] == r[1] * 2 for r in rows)
def test_default_start_end_returned() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"order_by": "timestamp",
"limit": 5,
"columns": ["timestamp"],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert data["start"] == "2024-01-01 00:00:00"
assert data["end"] == "2024-01-02 03:00:00"I'm adding a test for when "time_column" is none, and no start/end are provided.
If time_column is None, I'll skip setting start/end and leave them as None.
I'm ensuring no start/end values are sent or shown in UI when time_column is empty.
I'll update JS to show/hide start/end fields when time_column is an empty string.
root@ee1d42b3cdab:/workspace/scubaduck# grep -n "updateDisplayTypeUI" -n scubadu
ck/static/index.html
506:function updateDisplayTypeUI() {
547:graphTypeSel.addEventListener('change', updateDisplayTypeUI);
711: updateDisplayTypeUI();
1095: updateDisplayTypeUI();.html
function updateDisplayTypeUI() {
const prevType = displayType;
updateSelectedColumns(prevType);
const newType = graphTypeSel.value;
const showTable = newType === 'table';
const showTS = newType === 'timeseries';
document.getElementById('group_by_field').style.display = showTable || showTS
? 'flex' : 'none';
document.getElementById('aggregate_field').style.display = showTable || showTS
? 'flex' : 'none';
document.getElementById('show_hits_field').style.display = showTable ? 'flex'
: 'none';
document.getElementById('x_axis_field').style.display = showTS ? 'flex' : 'non
e';
document.getElementById('granularity_field').style.display = showTS ? 'flex' :
'none';
document.getElementById('fill_field').style.display = showTS ? 'flex' : 'none'
;
document.querySelectorAll('#column_groups .col-group').forEach(g => {
if (g.querySelector('.col-group-header').textContent.startsWith('Strings'))
{
g.style.display = showTable || showTS ? 'none' : '';
}
});
limitValues[prevType] = parseInt(limitInput.value, 10);
if (showTS && limitValues.timeseries === undefined) {
limitValues.timeseries = 7;
}
limitInput.value = limitValues[newType];
document.querySelectorAll('#column_groups input').forEach(cb => {
cb.checked = columnValues[newType].includes(cb.value);
});
if (showTS) {
document.querySelectorAll('#column_groups input').forEach(cb => {
if (isTimeColumn(cb.value) || isStringColumn(cb.value)) {
cb.checked = false;
}
});
document.getElementById('order_by').value = '';
}
updateSelectedColumns(newType);
displayType = newType;
}
orderDirBtn.addEventListener('click', () => {
orderDir = orderDir === 'ASC' ? 'DESC' : 'ASC';
updateOrderDirButton();
});
updateOrderDirButton();
graphTypeSel.addEventListener('change', updateDisplayTypeUI);I’m looking for functions like showTimeFields, especially related to '#time_column' events.
root@ee1d42b3cdab:/workspace/scubaduck# grep -n "time_column" -n scubaduck/stati
c/index.html | sed -n '40,140p'I'm searching for updateTimeColumn or any relevant events to handle time_column selection.
root@ee1d42b3cdab:/workspace/scubaduck# grep -n "#time_column" -n scubaduck/stat
ic/index.htmlI guess I need to check more lines around line 1080. Let's do that.
root@ee1d42b3cdab:/workspace/scubaduck# sed -n '1070,1100p' scubaduck/static/ind
ex.html
if (params.x_axis) sp.set('x_axis', params.x_axis);
if (params.granularity) sp.set('granularity', params.granularity);
if (params.fill) sp.set('fill', params.fill);
}
const qs = sp.toString();
return qs ? '?' + qs : '';
}
function applyParams(params) {
if (params.table) document.getElementById('table').value = params.table;
document.getElementById('time_column').value = params.time_column || defaultTi
meColumn;
if (params.time_unit) document.getElementById('time_unit').value = params.time
_unit;
document.getElementById('start').value = params.start || '';
document.getElementById('end').value = params.end || '';
if (params.order_by) {
document.getElementById('order_by').value = params.order_by;
}
orderDir = params.order_dir || 'ASC';
updateOrderDirButton();
if (params.limit !== undefined && params.limit !== null) {
document.getElementById('limit').value = params.limit;
limitValues[params.graph_type || 'samples'] = params.limit;
limitInput.dataset.setByUser = '1';
}
graphTypeSel.value = params.graph_type || 'samples';
updateDisplayTypeUI();
limitInput.value = limitValues[graphTypeSel.value];
if (params.x_axis) {
document.getElementById('x_axis').value = params.x_axis;
} else {
document.getElementById('x_axis').value = '';I should either create a new function or adjust updateDisplayTypeUI to hide these fields when time_column is empty.
root@ee1d42b3cdab:/workspace/scubaduck# grep -n "#start" -n scubaduck/static/ind
ex.htmlThe issue might be caused by the use of 'id="start"', not '#start'. Time to adjust.
root@ee1d42b3cdab:/workspace/scubaduck# grep -n "id=\"start\"" -n scubaduck/stat
ic/index.html
214: <input id="start" type="text" />I'll implement a listener that shows or hides the start/end fields based on selection.
root@ee1d42b3cdab:/workspace/scubaduck# grep -n "start'" -n scubaduck/static/ind
ex.html
1005: start: document.getElementById('start').value,
1053: if (params.start) sp.set('start', params.start);
1082: document.getElementById('start').value = params.start || '';
1152: if (sp.has('start')) params.start = sp.get('start');I'll add "(none)" to the time column select and implement visibility logic for Start/End fields.
I'll update the time field visibility logic and modify tests for new "(none)" option.
root@ee1d42b3cdab:/workspace/scubaduck# grep -n "start_end" -n tests/test_web.pyI'll verify if there are other visibility tests in the codebase.
root@ee1d42b3cdab:/workspace/scubaduck# grep -n "Start" tests/test_web.py | headI’ll look for any '#start' references to get things moving.
root@ee1d42b3cdab:/workspace/scubaduck# grep -n "#start" tests/test_web.py
30: page.fill("#start", start)
286: page.fill("#start", "2024-01-01 00:00:00")
302: page.fill("#start", "2024-01-01 00:00:00")
471: page.fill("#start", "2024-01-01 00:00:00")
549: page.click("#start-select div:text('-3 hours')")
550: assert page.input_value("#start") == "-3 hours"
612: page.fill("#start", "2024-01-01 00:00:00")
995: page.fill("#start", "2024-01-01 00:00:00")
1027: assert page.input_value("#start") == "2024-01-01 00:00:00"
1154: page.fill("#start", "2024-01-01 00:00:00")
1282: page.fill("#start", "2024-01-01 00:00:00")
1333: page.fill("#start", "2024-01-01 00:00:00")I see tests referencing start field. I need to check for "rel-dropdown" and related tests.
root@ee1d42b3cdab:/workspace/scubaduck# grep -n "rel-dropdown" -n tests/test_web
.pyI'm adding a test to check if setting the time column to "(none)" behaves as expected.
root@ee1d42b3cdab:/workspace/scubaduck# grep -n "time range" -n tests/test_web.p
y
487: assert any("start/end of the time range" in t for t in titles)I'm checking near line 487 for tests related to time-range.
assert path is not None
coords = [float(p.split(" ")[1]) for p in path.replace("M", "L").split("L")[
1:]]
assert max(coords) > min(coords)
def test_timeseries_multi_series(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#graph_type", state="attached")
select_value(page, "#graph_type", "timeseries")
page.click("text=Columns")
page.check("#column_groups input[value='value']")
page.click("text=Add Derived")
expr = page.query_selector("#derived_list .derived textarea")
assert expr
name_inp = page.query_selector("#derived_list .derived .d-name")
assert name_inp
name_inp.fill("value_2")
expr.fill("value * 2")
page.click("text=View Settings")
page.fill("#start", "2024-01-01 00:00:00")
page.fill("#end", "2024-01-03 00:00:00")
select_value(page, "#granularity", "1 hour")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
count = page.eval_on_selector_all("#chart path", "els => els.length")
assert count == 2
def test_help_and_alignment(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
titles = page.evaluate(
"Array.from(document.querySelectorAll('#settings .help')).map(e => e.tit
le)"
)
assert any("start/end of the time range" in t for t in titles)
text_align = page.evaluate(
"getComputedStyle(document.querySelector('#settings label')).textAlign"
)
assert text_align == "right"
def test_table_sorting(page: Any, server_url: str) -> None:
run_query(
page,
server_url,
start="2024-01-01 00:00:00",
end="2024-01-03 00:00:00",
order_by="timestamp",
order_dir="ASC",
limit=100,
)
# header alignment
align = page.evaluate(
"getComputedStyle(document.querySelector('#results th')).textAlign"
)
assert align == "right"
header = page.locator("#results th").nth(3)
def values() -> list[str]:
return page.locator("#results td:nth-child(4)").all_inner_texts()
orig_rows = values()
assert orig_rows == ["alice", "bob", "alice", "charlie"]
first_sql = page.evaluate("window.lastResults.sql")I'll check the bottom of tests/test_web.py for the relative start dropdown test.
root@ee1d42b3cdab:/workspace/scubaduck# grep -n "start-select" -n tests/test_web
.py
546: btn = page.query_selector('[data-target="start-select"]')
549: page.click("#start-select div:text('-3 hours')")root@ee1d42b3cdab:/workspace/scubaduck# sed -n '520,560p' tests/test_web.py
header.click()
assert values() == sorted(orig_rows)
assert header.inner_text().endswith("▲")
color = page.evaluate(
"getComputedStyle(document.querySelector('#results th:nth-child(4)')).co
lor"
)
assert "0, 0, 255" in color
assert page.evaluate("window.lastResults.sql") == first_sql
header.click()
assert values() == sorted(orig_rows, reverse=True)
assert header.inner_text().endswith("▼")
header.click()
assert values() == orig_rows
assert header.inner_text() == "user"
color = page.evaluate(
"getComputedStyle(document.querySelector('#results th:nth-child(4)')).co
lor"
)
assert "0, 0, 255" not in color
def test_relative_dropdown(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
btn = page.query_selector('[data-target="start-select"]')
assert btn
btn.click()
page.click("#start-select div:text('-3 hours')")
assert page.input_value("#start") == "-3 hours"
def test_end_dropdown_now(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
page.click('[data-target="end-select"]')
page.click("#end-select div:text('now')")
assert page.input_value("#end") == "now"I'll take a look at the sample.csv now to review its columns.
timestamp,event,value,user
2024-01-01 00:00:00,login,10,alice
2024-01-01 01:00:00,logout,20,bob
2024-01-02 00:00:00,login,30,alice
2024-01-02 03:00:00,login,40,charlieI’m planning updates to handle time_column: None; features include skipping filters for start/end.
I’ll need to activate the virtual environment to run the tests for this query.
root@ee1d42b3cdab:/workspace/scubaduck# source .venv/bin/activate
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck#I'm inspecting the server tests to check patterns and see how the filter can be verified.
server.py | head
174: sqlite_file = tmp_path / "events.sqlite"
175: import sqlite3
177: conn = sqlite3.connect(sqlite_file)
198: for db in (csv_file, sqlite_file, duckdb_file):
209:def test_sqlite_longvarchar(tmp_path: Path) -> None:
210: sqlite_file = tmp_path / "events.sqlite"
211: import sqlite3
213: conn = sqlite3.connect(sqlite_file)
223: app = server.create_app(sqlite_file)
240:def test_sqlite_bigint(tmp_path: Path) -> None:
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck#I’ll search for 'sql' to see if it's relevant.
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck# grep -n "data\[\"sql\"\]" -n
tests/test_server.py
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck#I haven't found tests checking 'sql' yet—let's search to be sure.
server.py | head
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck#I'll search for "sql" instead to see if it's present.
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck# grep -n "sql" -n tests/test_
server.py | head
174: sqlite_file = tmp_path / "events.sqlite"
175: import sqlite3
177: conn = sqlite3.connect(sqlite_file)
198: for db in (csv_file, sqlite_file, duckdb_file):
209:def test_sqlite_longvarchar(tmp_path: Path) -> None:
210: sqlite_file = tmp_path / "events.sqlite"
211: import sqlite3
213: conn = sqlite3.connect(sqlite_file)
223: app = server.create_app(sqlite_file)
240:def test_sqlite_bigint(tmp_path: Path) -> None:
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck#Scrolling through, I’m approaching line 174, where I might uncover earlier test code.
"start": "2024-01-01 00:00:00",
"end": "2024-01-02 00:00:00",
"order_by": "timestamp",
"order_dir": "ASC",
"limit": 10,
"columns": ["timestamp", "event", "value", "user"],
"filters": [],
}
def test_database_types(tmp_path: Path) -> None:
csv_file = tmp_path / "events.csv"
csv_file.write_text(Path("scubaduck/sample.csv").read_text())
sqlite_file = tmp_path / "events.sqlite"
import sqlite3
conn = sqlite3.connect(sqlite_file)
conn.execute(
"CREATE TABLE events (timestamp TEXT, event TEXT, value INTEGER, user TE
XT)"
)
with open(csv_file) as f:
next(f)
for line in f:
ts, ev, val, user = line.strip().split(",")
conn.execute(
"INSERT INTO events VALUES (?, ?, ?, ?)", (ts, ev, int(val), use
r)
)
conn.commit()
conn.close() # pyright: ignore[reportUnknownMemberType, reportAttributeAcce
ssIssue]
duckdb_file = tmp_path / "events.duckdb"
con = duckdb.connect(duckdb_file)
con.execute(
f"CREATE TABLE events AS SELECT * FROM read_csv_auto('{csv_file.as_posix
()}')"
)
con.close() # pyright: ignore[reportUnknownMemberType, reportAttributeAcces
sIssue]
for db in (csv_file, sqlite_file, duckdb_file):
app = server.create_app(db)
client = app.test_client()
payload = _make_payload()
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/js
on"
)
rows = rv.get_json()["rows"]
assert len(rows) == 3
def test_sqlite_longvarchar(tmp_path: Path) -> None:
sqlite_file = tmp_path / "events.sqlite"
import sqlite3
conn = sqlite3.connect(sqlite_file)
conn.execute(
"CREATE TABLE events (timestamp TEXT, url LONGVARCHAR, title VARCHAR(10)
)"
)
conn.execute(
"INSERT INTO events VALUES ('2024-01-01 00:00:00', 'https://a.com', 'Hom
e')"
)
conn.commit()
conn.close() # pyright: ignore[reportUnknownMemberType, reportAttributeAcce
ssIssue]
app = server.create_app(sqlite_file)
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-01 01:00:00",
"order_by": "timestamp",
"columns": ["timestamp", "url", "title"],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert data["rows"][0][1] == "https://a.com"
def test_sqlite_bigint(tmp_path: Path) -> None:_server.py
client = app.test_client()
base = {
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"order_by": "timestamp",
"limit": 100,
"columns": ["timestamp", "event", "value", "user"],
}
contains = {
**base,
"filters": [{"column": "user", "op": "contains", "value": "ali"}],
}
rv = client.post(
"/api/query", data=json.dumps(contains), content_type="application/json"
)
rows = rv.get_json()["rows"]
assert all("ali" in r[3] for r in rows)
regex = {
**base,
"filters": [{"column": "user", "op": "~", "value": "^a.*"}],
}
rv = client.post(
"/api/query", data=json.dumps(regex), content_type="application/json"
)
rows = rv.get_json()["rows"]
assert all(r[3].startswith("a") for r in rows)
assert len(rows) == 2
not_empty = {**base, "filters": [{"column": "user", "op": "!empty"}]}
rv = client.post(
"/api/query", data=json.dumps(not_empty), content_type="application/json
"
)
assert len(rv.get_json()["rows"]) == 4
def _make_payload() -> dict[str, object]:
return {
"table": "events",
"start": "2024-01-01 00:00:00",
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck#I'm getting closer, almost at the test_time_column_dropdown section now.
data = run_query(
page,
server_url,
start="2024-01-01 00:00:00",
end="2024-01-03 00:00:00",
order_by="timestamp",
limit=2,
)
assert len(data["rows"]) == 2
def test_time_column_dropdown(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#time_column option", state="attached")
options = page.locator("#time_column option").all_inner_texts()
assert "timestamp" in options
assert "value" in options
assert page.input_value("#time_column") == "timestamp"
def test_time_unit_dropdown(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#time_unit", state="attached")
opts = page.locator("#time_unit option").all_inner_texts()
assert "ms" in opts
assert page.input_value("#time_unit") == "s"
def test_x_axis_default_entry(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#graph_type", state="attached")
select_value(page, "#graph_type", "timeseries")
page.wait_for_selector("#x_axis option", state="attached")
options = page.locator("#x_axis option").all_inner_texts()
assert "(default)" in options
assert page.input_value("#x_axis") == ""
def test_simple_filter(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck#I’m planning to update checks around time_column, ensuring proper behavior with params and filters.
aduck/server.py
489: allow_time = False
492: allow_time = True
495: allow_time = False
496: if need_numeric or allow_time:
525: if allow_time and not (is_numeric or is_time):
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck#I'll focus on lines 480 to 520 to verify the details.
ctype = column_types.get(params.x_axis, "").upper()
is_time = any(t in ctype for t in ["TIMESTAMP", "DATE", "TIME"])
is_numeric = any(
t in ctype
for t in [
"INT",
"DECIMAL",
"REAL",
"DOUBLE",
"FLOAT",
"NUMERIC",
"HUGEINT",
]
)
if not (is_time or is_numeric):
return jsonify({"error": "x_axis must be a time column"}), 400
for col in params.columns:
if col not in valid_cols:
return jsonify({"error": f"Unknown column: {col}"}), 400
for col in params.group_by:
if col not in valid_cols:
return jsonify({"error": f"Unknown column: {col}"}), 400
if params.order_by and params.order_by not in valid_cols:
return jsonify({"error": f"Unknown column: {params.order_by}"}), 400
if params.group_by or params.graph_type == "timeseries":
agg = (params.aggregate or "avg").lower()
if agg.startswith("p") or agg == "sum":
need_numeric = True
allow_time = False
elif agg == "avg" or agg in {"min", "max"}:
need_numeric = False
allow_time = True
else:
need_numeric = False
allow_time = False
if need_numeric or allow_time:
for c in params.columns:
if c in params.group_by or c == params.x_axis:
continue
if c not in column_types:
continue
ctype = column_types.get(c, "").upper()
is_numeric = any(
t in ctype
for t in [
"INT",
"DECIMAL",
"REAL",
"DOUBLE",
"FLOAT",
"NUMERIC",
"HUGEINT",
]
)
is_time = "TIMESTAMP" in ctype or "DATE" in ctype or "TIME"
in ctype
if need_numeric and not is_numeric:
return (
jsonify(
{
"error": f"Aggregate {agg} cannot be applied
to column {c}",
}
),
400,
)
if allow_time and not (is_numeric or is_time):
return (
jsonify(
{
"error": f"Aggregate {agg} cannot be applied
to column {c}",
}
),
400,
)
if params.start is None or params.end is None:
axis = params.x_axis or params.time_column
row = cast(
tuple[datetime | None, datetime | None],
con.execute(
f'SELECT min({axis}), max({axis}) FROM "{params.table}"'
).fetchall()[0],(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck# sed -n '540,620p' scubaduck/
server.py
).fetchall()[0],
)
mn, mx = row
divisor = {
"s": 1,
"ms": 1000,
"us": 1_000_000,
"ns": 1_000_000_000,
}.get(params.time_unit, 1)
if isinstance(mn, (int, float)):
try:
mn = datetime.fromtimestamp(int(mn) / divisor, tz=timezone.u
tc)
except Exception:
return (
jsonify(
{
"error": (
f"Invalid time value {mn} for column {axis}"
f" with time_unit {params.time_unit}"
)
}
),
400,
)
if isinstance(mx, (int, float)):
try:
mx = datetime.fromtimestamp(int(mx) / divisor, tz=timezone.u
tc)
except Exception:
return (
jsonify(
{
"error": (
f"Invalid time value {mx} for column {axis}"
f" with time_unit {params.time_unit}"
)
}
),
400,
)
if params.start is None and mn is not None:
params.start = (
mn.strftime("%Y-%m-%d %H:%M:%S") if not isinstance(mn, str)
else mn
)
if params.end is None and mx is not None:
params.end = (
mx.strftime("%Y-%m-%d %H:%M:%S") if not isinstance(mx, str)
else mx
)
bucket_size: int | None = None
if params.graph_type == "timeseries":
bucket_size = _granularity_seconds(
params.granularity,
params.start if isinstance(params.start, str) else None,
params.end if isinstance(params.end, str) else None,
)
if (
params.limit is not None
and params.start is not None
and params.end is not None
):
try:
start_dt = dtparser.parse(params.start)
end_dt = dtparser.parse(params.end)
buckets = math.ceil(
(end_dt - start_dt).total_seconds() / bucket_size
)
if buckets > 1:
params.limit *= buckets
except Exception:
pass
sql = build_query(params, column_types)
try:
rows = con.execute(sql).fetchall()
except Exception as exc:
tb = traceback.format_exc()
print(f"Query failed:\n{sql}\n{tb}")
return (
jsonify({"sql": sql, "error": str(exc), "traceback": tb}),
400,
)(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck# sed -n '620,700p' scubaduck/
server.py
)
result: Dict[str, Any] = {"sql": sql, "rows": rows}
if params.start is not None:
result["start"] = str(params.start)
if params.end is not None:
result["end"] = str(params.end)
if bucket_size is not None:
result["bucket_size"] = bucket_size
return jsonify(result)
return app
app = create_app()
if __name__ == "__main__":
app.run(debug=True)
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck#I'll update the QueryParams dataclass, modify build_query, and adjust create_app for None checks.
@dataclass
class QueryParams:
start: str | None = None
end: str | None = None
order_by: str | None = None
order_dir: str = "ASC"
limit: int | None = None
columns: list[str] = field(default_factory=lambda: [])
filters: list[Filter] = field(default_factory=lambda: [])
derived_columns: dict[str, str] = field(default_factory=lambda: {})
graph_type: str = "samples"
group_by: list[str] = field(default_factory=lambda: [])
aggregate: str | None = None
show_hits: bool = False
x_axis: str | None = None
granularity: str = "Auto"
fill: str = "0"
table: str = "events"
time_column: str = "timestamp"
time_unit: str = "s"
def _load_database(path: Path) -> duckdb.DuckDBPyConnection:
if not path.exists():
raise FileNotFoundError(path)
ext = path.suffix.lower()
if ext == ".csv":
con = duckdb.connect()
con.execute(
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck#I need to update the dataclass to reflect the time_column as "timestamp".
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck# apply_patch<<'PATCH'
> *** Begin Patch
> *** Update File: scubaduck/server.py
> @@
> - table: str = "events"
> - time_column: str = "timestamp"
> + table: str = "events"
> + time_column: str | None = "timestamp"
> time_unit: str = "s"
> *** End Patch
> PATCH
Success. Updated the following files:
M scubaduck/server.py
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck#Time to update the build_query function—I'll apply the necessary patch now.
baduck/server.py
183:def build_query(params: QueryParams, column_types: Dict[str, str] | None = N
one) -> str:
244: inner_sql = build_query(inner_params, column_types)
611: sql = build_query(params, column_types)(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck# sed -n '240,272p' scubaduck/
server.py
derived_columns={},
order_by=None,
limit=None,
)
inner_sql = build_query(inner_params, column_types)
outer_select = ["t.*"] + [
f"{expr} AS {name}" for name, expr in params.derived_columns.items()
]
indented_inner = "\n".join(" " + line for line in inner_sql.splitline
s())
lines = [
f"SELECT {', '.join(outer_select)}",
"FROM (",
indented_inner,
") t",
]
if params.order_by:
lines.append(f"ORDER BY {params.order_by} {params.order_dir}")
elif params.graph_type == "timeseries":
lines.append("ORDER BY bucket")
if params.limit is not None:
lines.append(f"LIMIT {params.limit}")
return "\n".join(lines)
for name, expr in params.derived_columns.items():
select_parts.append(f"{expr} AS {name}")
select_clause = ", ".join(select_parts) if select_parts else "*"
lines = [f"SELECT {select_clause}", f'FROM "{params.table}"']
where_parts: list[str] = []
time_expr = _time_expr(params.time_column, column_types, params.time_unit)
if params.start:
where_parts.append(f"{time_expr} >= '{params.start}'")
if params.end:
where_parts.append(f"{time_expr} <= '{params.end}'")
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck#I'll update the code to skip computing time_expr if time_column is None.
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck# apply_patch<<'PATCH'
> *** Begin Patch
> *** Update File: scubaduck/server.py
> @@
> - time_expr = _time_expr(params.time_column, column_types, params.time_unit
)
> - if params.start:
> - where_parts.append(f"{time_expr} >= '{params.start}'")
> - if params.end:
> - where_parts.append(f"{time_expr} <= '{params.end}'")
> + if params.time_column:
> + time_expr = _time_expr(params.time_column, column_types, params.time_
unit)
> + else:
> + time_expr = None
> + if time_expr and params.start:
> + where_parts.append(f"{time_expr} >= '{params.start}'")
> + if time_expr and params.end:
> + where_parts.append(f"{time_expr} <= '{params.end}'")
> *** End Patch
> PATCH
Success. Updated the following files:
M scubaduck/server.py
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck#I'll adjust the start/end block earlier and focus on the lines that compute them.
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck# grep -n "SELECT min" -n scub
aduck/server.py
542: f'SELECT min({axis}), max({axis}) FROM "{params.table}"'(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck# sed -n '524,568p' scubaduck/
server.py
}
),
400,
)
if allow_time and not (is_numeric or is_time):
return (
jsonify(
{
"error": f"Aggregate {agg} cannot be applied
to column {c}",
}
),
400,
)
if params.start is None or params.end is None:
axis = params.x_axis or params.time_column
row = cast(
tuple[datetime | None, datetime | None],
con.execute(
f'SELECT min({axis}), max({axis}) FROM "{params.table}"'
).fetchall()[0],
)
mn, mx = row
divisor = {
"s": 1,
"ms": 1000,
"us": 1_000_000,
"ns": 1_000_000_000,
}.get(params.time_unit, 1)
if isinstance(mn, (int, float)):
try:
mn = datetime.fromtimestamp(int(mn) / divisor, tz=timezone.u
tc)
except Exception:
return (
jsonify(
{
"error": (
f"Invalid time value {mn} for column {axis}"
f" with time_unit {params.time_unit}"
)
}
),
400,
)
if isinstance(mx, (int, float)):
try:(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck# sed -n '568,620p' scubaduck/
server.py
try:
mx = datetime.fromtimestamp(int(mx) / divisor, tz=timezone.u
tc)
except Exception:
return (
jsonify(
{
"error": (
f"Invalid time value {mx} for column {axis}"
f" with time_unit {params.time_unit}"
)
}
),
400,
)
if params.start is None and mn is not None:
params.start = (
mn.strftime("%Y-%m-%d %H:%M:%S") if not isinstance(mn, str)
else mn
)
if params.end is None and mx is not None:
params.end = (
mx.strftime("%Y-%m-%d %H:%M:%S") if not isinstance(mx, str)
else mx
)
bucket_size: int | None = None
if params.graph_type == "timeseries":
bucket_size = _granularity_seconds(
params.granularity,
params.start if isinstance(params.start, str) else None,
params.end if isinstance(params.end, str) else None,
)
if (
params.limit is not None
and params.start is not None
and params.end is not None
):
try:
start_dt = dtparser.parse(params.start)
end_dt = dtparser.parse(params.end)
buckets = math.ceil(
(end_dt - start_dt).total_seconds() / bucket_size
)
if buckets > 1:
params.limit *= buckets
except Exception:
pass
sql = build_query(params, column_types)
try:
rows = con.execute(sql).fetchall()
except Exception as exc:
tb = traceback.format_exc()
print(f"Query failed:\n{sql}\n{tb}")
return (
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck#I'll adjust the condition to ensure the block runs only if time_column/x_axis are provided.
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck# apply_patch<<'PATCH'
> *** Begin Patch
> *** Update File: scubaduck/server.py
> @@
> - if params.start is None or params.end is None:
> - axis = params.x_axis or params.time_column
> + if (
> + params.start is None or params.end is None
> + ) and (params.x_axis or params.time_column):
> + axis = params.x_axis or params.time_column
> row = cast(
> tuple[datetime | None, datetime | None],
> con.execute(
> f'SELECT min({axis}), max({axis}) FROM "{params.table}"'
> ).fetchall()[0],
> *** End Patch
> PATCH
Success. Updated the following files:
M scubaduck/server.py
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck#I need to change every instance of params.time_column expecting a string.
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck# grep -n "time_column" -n scu
baduck/server.py
45: time_column: str | None = "timestamp"
188: x_axis = params.x_axis or params.time_column
268: if params.time_column:
269: time_expr = _time_expr(params.time_column, column_types, params.time
_unit)
427: time_column=payload.get("time_column", "timestamp"),
438: if params.time_column not in column_types:
439: return jsonify({"error": "Invalid time_column"}), 400
460: params.x_axis = params.time_column
539: ) and (params.x_axis or params.time_column):
540: axis = params.x_axis or params.time_column
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck#I'll focus on lines 427 to 460 to make the necessary changes.
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck# sed -n '412,468p' scubaduck/
server.py
start=start,
end=end,
order_by=payload.get("order_by"),
order_dir=payload.get("order_dir", "ASC"),
limit=payload.get("limit"),
columns=payload.get("columns", []),
derived_columns=payload.get("derived_columns", {}),
graph_type=payload.get("graph_type", "samples"),
group_by=payload.get("group_by", []),
aggregate=payload.get("aggregate"),
show_hits=payload.get("show_hits", False),
x_axis=payload.get("x_axis"),
granularity=payload.get("granularity", "Auto"),
fill=payload.get("fill", "0"),
table=payload.get("table", default_table),
time_column=payload.get("time_column", "timestamp"),
time_unit=payload.get("time_unit", "s"),
)
for f in payload.get("filters", []):
params.filters.append(Filter(f["column"], f["op"], f.get("value")))
if params.table not in tables:
return jsonify({"error": "Invalid table"}), 400
column_types = get_columns(params.table)
if params.time_column not in column_types:
return jsonify({"error": "Invalid time_column"}), 400
if params.time_unit not in {"s", "ms", "us", "ns"}:
return jsonify({"error": "Invalid time_unit"}), 400
if params.graph_type not in {"table", "timeseries"} and (
params.group_by or params.aggregate or params.show_hits
):
return (
jsonify(
{
"error": "group_by, aggregate and show_hits are only val
id for table or timeseries view"
}
),
400,
)
valid_cols = set(column_types.keys())
valid_cols.update(params.derived_columns.keys())
if params.graph_type == "timeseries":
if params.x_axis is None:
params.x_axis = params.time_column
if params.x_axis not in valid_cols:
return jsonify({"error": "Invalid x_axis"}), 400
ctype = column_types.get(params.x_axis, "").upper()
is_time = any(t in ctype for t in ["TIMESTAMP", "DATE", "TIME"])
is_numeric = any(
t in ctype
for t in [
"INT",
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck#I'll update the time_column handling to accept None and adjust subsequent x_axis logic.
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck# apply_patch<<'PATCH'
> *** Begin Patch
> *** Update File: scubaduck/server.py
> @@
> - time_column=payload.get("time_column", "timestamp"),
> + time_column=payload.get("time_column", "timestamp"),
> time_unit=payload.get("time_unit", "s"),
> )
> @@
> - if params.time_column not in column_types:
> - return jsonify({"error": "Invalid time_column"}), 400
> + if params.time_column and params.time_column not in column_types:
> + return jsonify({"error": "Invalid time_column"}), 400
> *** End Patch
> PATCH
Success. Updated the following files:
M scubaduck/server.py
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck#I'll update the timeseries x_axis to have a default value.
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck# grep -n "x_axis is None" -n
scubaduck/server.py
459: if params.x_axis is None:(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck# sed -n '456,468p' scubaduck/
server.py
valid_cols = set(column_types.keys())
valid_cols.update(params.derived_columns.keys())
if params.graph_type == "timeseries":
if params.x_axis is None:
params.x_axis = params.time_column
if params.x_axis not in valid_cols:
return jsonify({"error": "Invalid x_axis"}), 400
ctype = column_types.get(params.x_axis, "").upper()
is_time = any(t in ctype for t in ["TIMESTAMP", "DATE", "TIME"])
is_numeric = any(
t in ctype
for t in [
"INT",
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck#I'm adjusting the code to handle None for params.x_axis after checking params.time_column.
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck# apply_patch<<'PATCH'
> *** Begin Patch
> *** Update File: scubaduck/server.py
> @@
> - if params.graph_type == "timeseries":
> - if params.x_axis is None:
> - params.x_axis = params.time_column
> - if params.x_axis not in valid_cols:
> + if params.graph_type == "timeseries":
> + if params.x_axis is None:
> + params.x_axis = params.time_column
> + if params.x_axis is None or params.x_axis not in valid_cols:
> return jsonify({"error": "Invalid x_axis"}), 400
> *** End Patch
> PATCH
Success. Updated the following files:
M scubaduck/server.py
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck#The _time_expr function won't need changes. Reviewing edge cases, there's no issue if x_axis is None. To update JS, I’ll modify loadColumns function to add a default option for time_column. I'll patch index.html and adjust the relevant lines.
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck# sed -n '548,664p' scubaduck/
static/index.html
function loadColumns(table) {
return fetch('/api/columns?table=' + encodeURIComponent(table)).then(r => r.js
on()).then(cols => {
const orderSelect = document.getElementById('order_by');
const xAxisSelect = document.getElementById('x_axis');
const groupsEl = document.getElementById('column_groups');
const timeColumnSelect = document.getElementById('time_column');
orderSelect.innerHTML = '';
xAxisSelect.innerHTML = '';
const defOpt = document.createElement('option');
defOpt.value = '';
defOpt.textContent = '(default)';
xAxisSelect.appendChild(defOpt);
timeColumnSelect.innerHTML = '';
groupsEl.innerHTML = '';
allColumns.length = 0;
stringColumns.length = 0;
integerColumns.length = 0;
timeColumns.length = 0;
timeColumnOptions.length = 0;
baseColumns.length = 0;
baseStringColumns.length = 0;
baseIntegerColumns.length = 0;
baseTimeColumns.length = 0;
baseTimeColumnOptions.length = 0;
for (const k in columnTypes) delete columnTypes[k];
const groups = {
time: {name: 'Time', cols: [], ul: null},
integer: {name: 'Integers', cols: [], ul: null},
string: {name: 'Strings', cols: [], ul: null},
};
const heur = ['timestamp','created','created_at','event_time','time','date',
'occurred','happened','logged'];
let guess = null;
cols.forEach(c => {
const t = c.type.toUpperCase();
columnTypes[c.name] = c.type;
allColumns.push(c.name);
baseColumns.push(c.name);
let g = 'string';
const isNumeric = t.includes('INT') || t.includes('DECIMAL') || t.includes
('NUMERIC') || t.includes('REAL') || t.includes('DOUBLE') || t.includes('FLOAT')
|| t.includes('HUGEINT');
const isTimeType = t.includes('TIMESTAMP') || t.includes('DATE') || t.incl
udes('TIME');
if (isNumeric || isTimeType) {
timeColumnOptions.push(c.name);
baseTimeColumnOptions.push(c.name);
if (!guess && heur.some(h => c.name.toLowerCase().includes(h))) guess =
c.name;
}
if (isTimeType) {
g = 'time';
timeColumns.push(c.name);
baseTimeColumns.push(c.name);
} else if (isNumeric) {
g = 'integer';
}
if (g === 'string') {
stringColumns.push(c.name);
baseStringColumns.push(c.name);
} else if (g === 'integer') {
integerColumns.push(c.name);
baseIntegerColumns.push(c.name);
}
groups[g].cols.push(c.name);
if (g !== 'string') {
const o = document.createElement('option');
o.value = c.name;
o.textContent = c.name;
orderSelect.appendChild(o);
}
});
timeColumns.forEach(name => {
const o = document.createElement('option');
o.value = name;
o.textContent = name;
xAxisSelect.appendChild(o);
});
timeColumnOptions.forEach(name => {
const o = document.createElement('option');
o.value = name;
o.textContent = name;
timeColumnSelect.appendChild(o);
});
xAxisSelect.value = '';
defaultTimeColumn = guess || timeColumnOptions[0] || '';
Object.keys(groups).forEach(key => {
const g = groups[key];
const div = document.createElement('div');
div.className = 'col-group';
const header = document.createElement('div');
header.className = 'col-group-header';
header.appendChild(document.createTextNode(g.name + ': '));
const links = document.createElement('span');
links.className = 'links';
const allBtn = document.createElement('a');
allBtn.href = '#';
allBtn.textContent = 'All';
const noneBtn = document.createElement('a');
noneBtn.href = '#';
noneBtn.textContent = 'None';
links.appendChild(allBtn);
links.appendChild(noneBtn);
header.appendChild(links);
div.appendChild(header);
const ul = document.createElement('ul');
g.ul = ul;
g.cols.forEach(name => {
const li = document.createElement('li');
const label = document.createElement('label');
const cb = document.createElement('input');
cb.type = 'checkbox';
cb.value = name;
cb.checked = true;
cb.addEventListener('change', updateSelectedColumns);
label.appendChild(cb);
label.appendChild(document.createTextNode(' ' + name));
li.appendChild(label);
ul.appendChild(li);
});
allBtn.addEventListener('click', e => {
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck#I'll add a '(none)' option to TimeColumnSelect and call updateTimeFieldVisibility after loadColumns.
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck# grep -n "updateTimeFieldVisi
bility" -n scubaduck/static/index.html
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck#I'll add the new function near updateDisplayTypeUI, after updateOrderDirButton. Let's find context.
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck# grep -n "updateOrderDirButto
n" -n scubaduck/static/index.html
502:function updateOrderDirButton() {
544: updateOrderDirButton();
546:updateOrderDirButton();
1088: updateOrderDirButton();
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck#I'll review the lines between 500 and 548.
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck# sed -n '498,548p' scubaduck/
static/index.html
sidebarResizer.addEventListener('mousedown', startSidebarDrag);
let orderDir = 'ASC';
const orderDirBtn = document.getElementById('order_dir');
const graphTypeSel = document.getElementById('graph_type');
function updateOrderDirButton() {
orderDirBtn.textContent = orderDir + (orderDir === 'ASC' ? ' \u25B2' : ' \u25B
C');
}
function updateDisplayTypeUI() {
const prevType = displayType;
updateSelectedColumns(prevType);
const newType = graphTypeSel.value;
const showTable = newType === 'table';
const showTS = newType === 'timeseries';
document.getElementById('group_by_field').style.display = showTable || showTS
? 'flex' : 'none';
document.getElementById('aggregate_field').style.display = showTable || showTS
? 'flex' : 'none';
document.getElementById('show_hits_field').style.display = showTable ? 'flex'
: 'none';
document.getElementById('x_axis_field').style.display = showTS ? 'flex' : 'non
e';
document.getElementById('granularity_field').style.display = showTS ? 'flex' :
'none';
document.getElementById('fill_field').style.display = showTS ? 'flex' : 'none'
;
document.querySelectorAll('#column_groups .col-group').forEach(g => {
if (g.querySelector('.col-group-header').textContent.startsWith('Strings'))
{
g.style.display = showTable || showTS ? 'none' : '';
}
});
limitValues[prevType] = parseInt(limitInput.value, 10);
if (showTS && limitValues.timeseries === undefined) {
limitValues.timeseries = 7;
}
limitInput.value = limitValues[newType];
document.querySelectorAll('#column_groups input').forEach(cb => {
cb.checked = columnValues[newType].includes(cb.value);
});
if (showTS) {
document.querySelectorAll('#column_groups input').forEach(cb => {
if (isTimeColumn(cb.value) || isStringColumn(cb.value)) {
cb.checked = false;
}
});
document.getElementById('order_by').value = '';
}
updateSelectedColumns(newType);
displayType = newType;
}
orderDirBtn.addEventListener('click', () => {
orderDir = orderDir === 'ASC' ? 'DESC' : 'ASC';
updateOrderDirButton();
});
updateOrderDirButton();
graphTypeSel.addEventListener('change', updateDisplayTypeUI);
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck#I'm adding updateTimeFieldVisibility() after updateDisplayTypeUI and calling it in loadColumns and applyParams().
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck# apply_patch<<'PATCH'
> *** Begin Patch
> *** Update File: scubaduck/static/index.html
> @@
> function updateDisplayTypeUI() {
> @@
> updateSelectedColumns(newType);
> displayType = newType;
> }
> +function updateTimeFieldVisibility() {
> + const show = document.getElementById('time_column').value !== '';
> + document.getElementById('start').closest('.field').style.display = show
> + ? 'flex'
> + : 'none';
> + document.getElementById('end').closest('.field').style.display = show
> + ? 'flex'
> + : 'none';
> +}
> orderDirBtn.addEventListener('click', () => {
> *** End Patch
> PATCH
Success. Updated the following files:
M scubaduck/static/index.html
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck#I'll add an event listener for the time_column change after the updateOrderDirButton.
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck# grep -n "time_column'" -n sc
ubaduck/static/index.html
543: const show = document.getElementById('time_column').value !== '';
563: const timeColumnSelect = document.getElementById('time_column');
803: const sel = document.getElementById('time_column').value;
1012: time_column: document.getElementById('time_column').value,
1060: if (params.time_column) sp.set('time_column', params.time_column);
1089: document.getElementById('time_column').value = params.time_column || defa
ultTimeColumn;
1159: if (sp.has('time_column')) params.time_column = sp.get('time_column');
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck#I’ll add the event listener after defining updateTimeFieldVisibility and update it near lines 546-547.
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck# sed -n '540,560p' scubaduck/
static/index.html
displayType = newType;
}
function updateTimeFieldVisibility() {
const show = document.getElementById('time_column').value !== '';
document.getElementById('start').closest('.field').style.display = show
? 'flex'
: 'none';
document.getElementById('end').closest('.field').style.display = show
? 'flex'
: 'none';
}
orderDirBtn.addEventListener('click', () => {
orderDir = orderDir === 'ASC' ? 'DESC' : 'ASC';
updateOrderDirButton();
});
updateOrderDirButton();
graphTypeSel.addEventListener('change', updateDisplayTypeUI);
function loadColumns(table) {
return fetch('/api/columns?table=' + encodeURIComponent(table)).then(r => r.js
on()).then(cols => {
const orderSelect = document.getElementById('order_by');
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck#I'll add the event listener after updateOrderDirButton(); and then patch it.
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck# apply_patch<<'PATCH'
> *** Begin Patch
> *** Update File: scubaduck/static/index.html
> @@
> graphTypeSel.addEventListener('change', updateDisplayTypeUI);
> +document.getElementById('time_column').addEventListener('change', updateTimeF
ieldVisibility);
> *** End Patch
> PATCH
Success. Updated the following files:
M scubaduck/static/index.html
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck#I’ll update loadColumns() to include a (none) option and call updateTimeFieldVisibility.
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck# sed -n '560,648p' scubaduck/
static/index.html
return fetch('/api/columns?table=' + encodeURIComponent(table)).then(r => r.js
on()).then(cols => {
const orderSelect = document.getElementById('order_by');
const xAxisSelect = document.getElementById('x_axis');
const groupsEl = document.getElementById('column_groups');
const timeColumnSelect = document.getElementById('time_column');
orderSelect.innerHTML = '';
xAxisSelect.innerHTML = '';
const defOpt = document.createElement('option');
defOpt.value = '';
defOpt.textContent = '(default)';
xAxisSelect.appendChild(defOpt);
timeColumnSelect.innerHTML = '';
groupsEl.innerHTML = '';
allColumns.length = 0;
stringColumns.length = 0;
integerColumns.length = 0;
timeColumns.length = 0;
timeColumnOptions.length = 0;
baseColumns.length = 0;
baseStringColumns.length = 0;
baseIntegerColumns.length = 0;
baseTimeColumns.length = 0;
baseTimeColumnOptions.length = 0;
for (const k in columnTypes) delete columnTypes[k];
const groups = {
time: {name: 'Time', cols: [], ul: null},
integer: {name: 'Integers', cols: [], ul: null},
string: {name: 'Strings', cols: [], ul: null},
};
const heur = ['timestamp','created','created_at','event_time','time','date',
'occurred','happened','logged'];
let guess = null;
cols.forEach(c => {
const t = c.type.toUpperCase();
columnTypes[c.name] = c.type;
allColumns.push(c.name);
baseColumns.push(c.name);
let g = 'string';
const isNumeric = t.includes('INT') || t.includes('DECIMAL') || t.includes
('NUMERIC') || t.includes('REAL') || t.includes('DOUBLE') || t.includes('FLOAT')
|| t.includes('HUGEINT');
const isTimeType = t.includes('TIMESTAMP') || t.includes('DATE') || t.incl
udes('TIME');
if (isNumeric || isTimeType) {
timeColumnOptions.push(c.name);
baseTimeColumnOptions.push(c.name);
if (!guess && heur.some(h => c.name.toLowerCase().includes(h))) guess =
c.name;
}
if (isTimeType) {
g = 'time';
timeColumns.push(c.name);
baseTimeColumns.push(c.name);
} else if (isNumeric) {
g = 'integer';
}
if (g === 'string') {
stringColumns.push(c.name);
baseStringColumns.push(c.name);
} else if (g === 'integer') {
integerColumns.push(c.name);
baseIntegerColumns.push(c.name);
}
groups[g].cols.push(c.name);
if (g !== 'string') {
const o = document.createElement('option');
o.value = c.name;
o.textContent = c.name;
orderSelect.appendChild(o);
}
});
timeColumns.forEach(name => {
const o = document.createElement('option');
o.value = name;
o.textContent = name;
xAxisSelect.appendChild(o);
});
timeColumnOptions.forEach(name => {
const o = document.createElement('option');
o.value = name;
o.textContent = name;
timeColumnSelect.appendChild(o);
});
xAxisSelect.value = '';
defaultTimeColumn = guess || timeColumnOptions[0] || '';
Object.keys(groups).forEach(key => {
const g = groups[key];
const div = document.createElement('div');
div.className = 'col-group';
const header = document.createElement('div');
header.className = 'col-group-header';
header.appendChild(document.createTextNode(g.name + ': '));
const links = document.createElement('span');
links.className = 'links';
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck#I'll patch by inserting (none) after clearing timeColumnSelect but before iterating timeColumnOptions.
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck# apply_patch<<'PATCH'
> *** Begin Patch
> *** Update File: scubaduck/static/index.html
> @@
> - timeColumnSelect.innerHTML = '';
> + timeColumnSelect.innerHTML = '';
> + const noneOpt = document.createElement('option');
> + noneOpt.value = '';
> + noneOpt.textContent = '(none)';
> + timeColumnSelect.appendChild(noneOpt);
> *** End Patch
> PATCH
Success. Updated the following files:
M scubaduck/static/index.html
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck#Setting defaultTimeColumn to guess, or the first time column. Time field visibility needs updating after columns load.
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck# grep -n "defaultTimeColumn"
-n scubaduck/static/index.html
370:let defaultTimeColumn = '';
643: defaultTimeColumn = guess || timeColumnOptions[0] || '';
1094: document.getElementById('time_column').value = params.time_column || defa
ultTimeColumn;
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck#I'll add the updateTimeFieldVisibility call after the defaultTimeColumn line and patch it.
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck# sed -n '636,650p' scubaduck/
static/index.html
timeColumnOptions.forEach(name => {
const o = document.createElement('option');
o.value = name;
o.textContent = name;
timeColumnSelect.appendChild(o);
});
xAxisSelect.value = '';
defaultTimeColumn = guess || timeColumnOptions[0] || '';
Object.keys(groups).forEach(key => {
const g = groups[key];
const div = document.createElement('div');
div.className = 'col-group';
const header = document.createElement('div');
header.className = 'col-group-header';
header.appendChild(document.createTextNode(g.name + ': '));
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck#Right after the defaultTimeColumn line, I'll call the updateTimeFieldVisibility function.
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck# apply_patch<<'PATCH'
> *** Begin Patch
> *** Update File: scubaduck/static/index.html
> @@
> timeColumnOptions.forEach(name => {
> const o = document.createElement('option');
> o.value = name;
> o.textContent = name;
> timeColumnSelect.appendChild(o);
> });
> xAxisSelect.value = '';
> defaultTimeColumn = guess || timeColumnOptions[0] || '';
> + updateTimeFieldVisibility();
> *** End Patch
> PATCH
Success. Updated the following files:
M scubaduck/static/index.html
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck#I'll call updateTimeFieldVisibility after applyParams, ensuring time_column changes are correctly handled.
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck# grep -n "applyParams" -n scu
baduck/static/index.html
734: applyParams(parseSearch());
1093:function applyParams(params) {
1194: applyParams(params);
1200: applyParams(params);
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck#I'm opening around line 1093 to take a closer look.
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck# sed -n '1088,1108p' scubaduc
k/static/index.html
}
const qs = sp.toString();
return qs ? '?' + qs : '';
}
function applyParams(params) {
if (params.table) document.getElementById('table').value = params.table;
document.getElementById('time_column').value = params.time_column || defaultTi
meColumn;
if (params.time_unit) document.getElementById('time_unit').value = params.time
_unit;
document.getElementById('start').value = params.start || '';
document.getElementById('end').value = params.end || '';
if (params.order_by) {
document.getElementById('order_by').value = params.order_by;
}
orderDir = params.order_dir || 'ASC';
updateOrderDirButton();
if (params.limit !== undefined && params.limit !== null) {
document.getElementById('limit').value = params.limit;
limitValues[params.graph_type || 'samples'] = params.limit;
limitInput.dataset.setByUser = '1';
}
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck#I’ll add a call to updateTimeFieldVisibility after setting time_column.
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck# apply_patch<<'PATCH'
> *** Begin Patch
> *** Update File: scubaduck/static/index.html
> @@
> if (params.table) document.getElementById('table').value = params.table;
> document.getElementById('time_column').value = params.time_column || defaul
tTimeColumn;
> + updateTimeFieldVisibility();
> if (params.time_unit) document.getElementById('time_unit').value = params.t
ime_unit;
> *** End Patch
> PATCH
Success. Updated the following files:
M scubaduck/static/index.html
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck#I’m thinking of calling updateTimeFieldVisibility after setting the time_column, but only once, after page load.
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck# tail -n 20 scubaduck/static/
index.html
msg += '\n' + err.traceback;
}
}
view.innerHTML = `<pre id="error-message">${msg}</pre>`;
document.getElementById('query_info').textContent = '';
}
function setSelectValue(selector, value) {
const el = typeof selector === 'string' ? document.querySelector(selector) : s
elector;
if (el) {
const select = el.tagName === 'SELECT' ? el : el.querySelector('select');
if (select) {
select.value = value;
select.dispatchEvent(new Event('change'));
}
}
}
</script>
</body>
</html>
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck#I’ll call updateTimeFieldVisibility after initialization, right after setting up event listeners.
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck# grep -n "graphTypeSel.addEve
ntListener" -n scubaduck/static/index.html
556:graphTypeSel.addEventListener('change', updateDisplayTypeUI);(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck# sed -n '550,560p' scubaduck/
static/index.html
}
orderDirBtn.addEventListener('click', () => {
orderDir = orderDir === 'ASC' ? 'DESC' : 'ASC';
updateOrderDirButton();
});
updateOrderDirButton();
graphTypeSel.addEventListener('change', updateDisplayTypeUI);
document.getElementById('time_column').addEventListener('change', updateTimeFiel
dVisibility);
function loadColumns(table) {
return fetch('/api/columns?table=' + encodeURIComponent(table)).then(r => r.js
on()).then(cols => {
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck#I'll add updateTimeFieldVisibility(); after the event listener.
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck# apply_patch<<'PATCH'
> *** Begin Patch
> *** Update File: scubaduck/static/index.html
> @@
> graphTypeSel.addEventListener('change', updateDisplayTypeUI);
> document.getElementById('time_column').addEventListener('change', updateTimeF
ieldVisibility);
> +updateTimeFieldVisibility();
> *** End Patch
> PATCH
Success. Updated the following files:
M scubaduck/static/index.html
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck#I'll update collectParams to remove start/end if time_column is empty, but no URL change needed. No updates in paramsToSearch either since they aren't added when params.start is ''.
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck# sed -n '108,118p' tests/test
_web.py
assert len(data["rows"]) == 2
def test_time_column_dropdown(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#time_column option", state="attached")
options = page.locator("#time_column option").all_inner_texts()
assert "timestamp" in options
assert "value" in options
assert page.input_value("#time_column") == "timestamp"
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck#I'll add assert "(none)" in options and possibly include a new test after.
> *** Begin Patch
> *** Update File: tests/test_web.py
> @@
> def test_time_column_dropdown(page: Any, server_url: str) -> None:
> page.goto(server_url)
> page.wait_for_selector("#time_column option", state="attached")
> options = page.locator("#time_column option").all_inner_texts()
> + assert "(none)" in options
> assert "timestamp" in options
> assert "value" in options
> assert page.input_value("#time_column") == "timestamp"
> *** End Patch
> PATCH
Success. Updated the following files:
M tests/test_web.py
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck#I'll add a test to hide start/end if time_column is none, based on existing ones.
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck# grep -n "show_hits_field" -n
tests/test_web.py
223: assert page.is_visible("#show_hits_field")
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck#I'll check around line 210 to find the test call for test_graph_type_table_fields.
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck# sed -n '216,236p' tests/test
_web.py
def test_graph_type_table_fields(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#graph_type", state="attached")
select_value(page, "#graph_type", "table")
assert page.is_visible("#group_by_field")
assert page.is_visible("#aggregate_field")
assert page.is_visible("#show_hits_field")
page.click("text=Columns")
assert not page.is_visible("text=Strings:")
def test_graph_type_timeseries_fields(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#graph_type", state="attached")
select_value(page, "#graph_type", "timeseries")
assert page.is_visible("#group_by_field")
assert page.is_visible("#aggregate_field")
assert page.is_visible("#x_axis_field")
assert page.is_visible("#granularity_field")
assert page.is_visible("#fill_field")
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck#I'll add a test to check if the start and end fields are hidden after selecting an empty time column. I'll also verify the filter time_range is unaffected afterward.
from typing import Any
def select_value(page: Any, selector: str, value: str) -> None:
page.evaluate(
"arg => setSelectValue(arg.sel, arg.val)",
{"sel": selector, "val": value},
)
def run_query(
page: Any,
url: str,
*,
start: str | None = None,
end: str | None = None,
order_by: str | None = None,
order_dir: str | None = "ASC",
limit: int | None = None,
group_by: list[str] | None = None,
aggregate: str | None = None,
) -> dict[str, Any]:
page.goto(url)
page.wait_for_selector("#order_by option", state="attached")
page.wait_for_selector("#order_dir", state="attached")
page.wait_for_function("window.lastResults !== undefined")
if start is not None:
page.fill("#start", start)
if end is not None:
page.fill("#end", end)
if order_by is not None:
select_value(page, "#order_by", order_by)
if order_dir is not None and order_dir == "DESC":
page.click("#order_dir")
if limit is not None:
page.fill("#limit", str(limit))
if group_by is not None:
select_value(page, "#graph_type", "table")
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck#I'll modify run_query to accept a time_column parameter, then patch the test.
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck# sed -n '4,26p' tests/test_we
b.py
def select_value(page: Any, selector: str, value: str) -> None:
page.evaluate(
"arg => setSelectValue(arg.sel, arg.val)",
{"sel": selector, "val": value},
)
def run_query(
page: Any,
url: str,
*,
start: str | None = None,
end: str | None = None,
order_by: str | None = None,
order_dir: str | None = "ASC",
limit: int | None = None,
group_by: list[str] | None = None,
aggregate: str | None = None,
) -> dict[str, Any]:
page.goto(url)
page.wait_for_selector("#order_by option", state="attached")page.wait_for_selector("#order_by option", state="attached")
page.wait_for_selector("#order_dir", state="attached")
page.wait_for_function("window.lastResults !== undefined")
if start is not None:
page.fill("#start", start)
if end is not None:
page.fill("#end", end)
if order_by is not None:
select_value(page, "#order_by", order_by)
if order_dir is not None and order_dir == "DESC":
page.click("#order_dir")
if limit is not None:
page.fill("#limit", str(limit))
if group_by is not None:
select_value(page, "#graph_type", "table")
page.evaluate(
"g => { groupBy.chips = g; groupBy.renderChips(); }",
group_by,
)
if aggregate is not None:
select_value(page, "#graph_type", "table")
select_value(page, "#aggregate", aggregate)
if page.input_value("#graph_type") != "samples":
page.click("text=Columns")
page.wait_for_selector("#column_groups input", state="attached")
if not page.is_checked("#column_groups input[value='value']"):
page.check("#column_groups input[value='value']")
order_col = order_by or page.input_value("#order_by")
if order_col and not page.is_checked(
f"#column_groups input[value='{order_col}']"
):
if page.query_selector(f"#column_groups input[value='{order_col}']")
:
page.check(f"#column_groups input[value='{order_col}']")
page.click("text=View Settings")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
return page.evaluate("window.lastResults")
def test_range_filters(page: Any, server_url: str) -> None:
data = run_query(
page,
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck#I'll add a time_column parameter to the test and ensure it's not filtered by time.
> *** Begin Patch
> *** Update File: tests/test_web.py
> @@
> def test_time_column_dropdown(page: Any, server_url: str) -> None:
> @@
> assert page.input_value("#time_column") == "timestamp"
> +
> +
> +def test_time_column_none_hides_range(page: Any, server_url: str) -> None:
> + page.goto(server_url)
> + page.wait_for_selector("#time_column option", state="attached")
> + select_value(page, "#time_column", "")
> + assert page.is_hidden("#start")
> + assert page.is_hidden("#end")
> + page.evaluate("window.lastResults = undefined")
> + page.click("text=Dive")
> + page.wait_for_function("window.lastResults !== undefined")
> + data = page.evaluate("window.lastResults")
> + assert len(data["rows"]) == 4
> + assert "start" not in data and "end" not in data
> *** End Patch
> PATCH
Success. Updated the following files:
M tests/test_web.py
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck#I'll locate and add tests for time_column none, near default_start_end_returned. Let's proceed!
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck# grep -n "default_start_end_r
eturned" -n tests/test_server.py
775:def test_default_start_end_returned() -> None:
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck#I'll check approximately 760 lines of code to gather what's needed.
_server.py
"graph_type": "timeseries",
"granularity": "1 hour",
"limit": 7,
"columns": ["value"],
"derived_columns": {"derived_1": "value * 2"},
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
rows = data["rows"]
assert all(r[2] == r[1] * 2 for r in rows)
def test_default_start_end_returned() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"order_by": "timestamp",
"limit": 5,
"columns": ["timestamp"],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert data["start"] == "2024-01-01 00:00:00"
assert data["end"] == "2024-01-02 03:00:00"I'll add a test based on the current one, verifying it matches the 4 rows from sample.csv.
> *** Begin Patch
> *** Update File: tests/test_server.py
> @@
> def test_default_start_end_returned() -> None:
> @@
> assert data["end"] == "2024-01-02 03:00:00"
> +
> +
> +def test_time_column_none_no_time_filter() -> None:
> + app = server.app
> + client = app.test_client()
> + payload = {
> + "table": "events",
> + "columns": ["timestamp", "event"],
> + "time_column": "",
> + }
> + rv = client.post(
> + "/api/query", data=json.dumps(payload), content_type="application/jso
n"
> + )
> + data = rv.get_json()
> + assert rv.status_code == 200
> + assert "start" not in data and "end" not in data
> + assert len(data["rows"]) == 4
> *** End Patch
> PATCH
Success. Updated the following files:
M tests/test_server.py
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck#I'll run ruff format to check the changes I made to the tests.
.py scubaduck/static/index.html tests/test_web.py tests/test_server.py >/tmp/for
mat.log && tail -n 20 /tmp/format.log
error: Failed to parse scubaduck/static/index.html:1:1: Expected a statement
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck#Ruff only works for Python files, so I'll stick to formatting those.
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck# ruff format scubaduck/server
.py tests/test_web.py tests/test_server.py >/tmp/format.log && tail -n 20 /tmp/f
ormat.log
3 files left unchanged
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck#I'll run a quick ruff check next to spot any issues.
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck# ruff check scubaduck/server.
py scubaduck/static/index.html tests/test_web.py tests/test_server.py >/tmp/ruff
.log && tail -n 20 /tmp/ruff.log
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck#...
scubaduck/static/index.html:924:7: SyntaxError: Expected a statement
|
922 | <option value="numeric">Numeric</option>
923 | </select>
924 | <input class="d-name" type="text">
| ^
925 | <button type="button" class="remove" onclick="removeDerived(this)">✖
</button>
926 | </div>
|
scubaduck/static/index.html:924:14: SyntaxError: Compound statements are not all
owed on the same line as simple statements
|
922 | <option value="numeric">Numeric</option>
923 | </select>
924 | <input class="d-name" type="text">
| ^
925 | <button type="button" class="remove" onclick="removeDerived(this)">✖
</button>
926 | </div>
|
scubaduck/static/index.html:924:19: SyntaxError: Expected an identifier
|
922 | <option value="numeric">Numeric</option>
923 | </select>
924 | <input class="d-name" type="text">
| ^
925 | <button type="button" class="remove" onclick="removeDerived(this)">✖
</button>
926 | </div>
|
scubaduck/static/index.html:924:29: SyntaxError: Simple statements must be separ
ated by newlines or semicolons
|
922 | <option value="numeric">Numeric</option>
923 | </select>
924 | <input class="d-name" type="text">
| ^
925 | <button type="button" class="remove" onclick="removeDerived(this)">✖
</button>
926 | </div>
|
scubaduck/static/index.html:924:41: SyntaxError: Expected an expression
|
922 | <option value="numeric">Numeric</option>
923 | </select>
924 | <input class="d-name" type="text">
| ^
925 | <button type="button" class="remove" onclick="removeDerived(this)">✖
</button>
926 | </div>
927 | <label><input type="checkbox" class="d-use" checked> Include in Query<
/label>
|
scubaduck/static/index.html:925:7: SyntaxError: Expected a statement
|
923 | </select>
924 | <input class="d-name" type="text">
925 | <button type="button" class="remove" onclick="removeDerived(this)">✖
</button>
| ^
926 | </div>
927 | <label><input type="checkbox" class="d-use" checked> Include in Query<
/label>
|
scubaduck/static/index.html:925:15: SyntaxError: Simple statements must be separ
ated by newlines or semicolons
|
923 | </select>
924 | <input class="d-name" type="text">
925 | <button type="button" class="remove" onclick="removeDerived(this)">✖
</button>
| ^
926 | </div>
927 | <label><input type="checkbox" class="d-use" checked> Include in Query<
/label>
|
scubaduck/static/index.html:925:29: SyntaxError: Compound statements are not all
owed on the same line as simple statements
|
923 | </select>
924 | <input class="d-name" type="text">
925 | <button type="button" class="remove" onclick="removeDerived(this)">✖
</button>
| ^
926 | </div>
927 | <label><input type="checkbox" class="d-use" checked> Include in Query<
/label>
|
scubaduck/static/index.html:925:34: SyntaxError: Expected an identifier
|
923 | </select>
924 | <input class="d-name" type="text">
925 | <button type="button" class="remove" onclick="removeDerived(this)">✖
</button>
| ^
926 | </div>
927 | <label><input type="checkbox" class="d-use" checked> Include in Query<
/label>
|
scubaduck/static/index.html:925:44: SyntaxError: Simple statements must be separ
ated by newlines or semicolons
|
923 | </select>
924 | <input class="d-name" type="text">
925 | <button type="button" class="remove" onclick="removeDerived(this)">✖
</button>
| ^
926 | </div>
927 | <label><input type="checkbox" class="d-use" checked> Include in Query<
/label>
|
scubaduck/static/index.html:925:74: SyntaxError: Got unexpected token ✖
...
960 | allColumns.splice(0, allColumns.length, ...baseColumns);
|
scubaduck/static/index.html:959:10: SyntaxError: Simple statements must be separ
ated by newlines or semicolons
|
957 | }
958 |
959 | function refreshDerivedColumns() {
| ^
960 | allColumns.splice(0, allColumns.length, ...baseColumns);
961 | stringColumns.splice(0, stringColumns.length, ...baseStringColumns);
|
scubaduck/static/index.html:959:34: SyntaxError: Simple statements must be separ
ated by newlines or semicolons
|
957 | }
958 |
959 | function refreshDerivedColumns() {
| ^
960 | allColumns.splice(0, allColumns.length, ...baseColumns);
961 | stringColumns.splice(0, stringColumns.length, ...baseStringColumns);
|
scubaduck/static/index.html:960:46: SyntaxError: Expected ',', found name
|
959 | function refreshDerivedColumns() {
960 | allColumns.splice(0, allColumns.length, ...baseColumns);
| ^
961 | stringColumns.splice(0, stringColumns.length, ...baseStringColumns);
962 | integerColumns.splice(0, integerColumns.length, ...baseIntegerColumns);
|
scubaduck/static/index.html:960:58: SyntaxError: Expected an expression or a '}'
|
959 | function refreshDerivedColumns() {
960 | allColumns.splice(0, allColumns.length, ...baseColumns);
| ^
961 | stringColumns.splice(0, stringColumns.length, ...baseStringColumns);
962 | integerColumns.splice(0, integerColumns.length, ...baseIntegerColumns);
|
scubaduck/static/index.html:961:52: SyntaxError: Expected ',', found name
|
959 | function refreshDerivedColumns() {
960 | allColumns.splice(0, allColumns.length, ...baseColumns);
961 | stringColumns.splice(0, stringColumns.length, ...baseStringColumns);
| ^
962 | integerColumns.splice(0, integerColumns.length, ...baseIntegerColumns);
963 | timeColumns.splice(0, timeColumns.length, ...baseTimeColumns);
|
scubaduck/static/index.html:961:70: SyntaxError: Expected ',', found ';'
|
959 | function refreshDerivedColumns() {
960 | allColumns.splice(0, allColumns.length, ...baseColumns);
961 | stringColumns.splice(0, stringColumns.length, ...baseStringColumns);
| ^
962 | integerColumns.splice(0, integerColumns.length, ...baseIntegerColumns);
963 | timeColumns.splice(0, timeColumns.length, ...baseTimeColumns);
|
scubaduck/static/index.html:962:54: SyntaxError: Expected ',', found name
|
960 | allColumns.splice(0, allColumns.length, ...baseColumns);
961 | stringColumns.splice(0, stringColumns.length, ...baseStringColumns);
962 | integerColumns.splice(0, integerColumns.length, ...baseIntegerColumns);
| ^
963 | timeColumns.splice(0, timeColumns.length, ...baseTimeColumns);
964 | timeColumnOptions.splice(0, timeColumnOptions.length, ...baseTimeColumnO
ptions);
|
scubaduck/static/index.html:962:73: SyntaxError: Expected ',', found ';'
|
960 | allColumns.splice(0, allColumns.length, ...baseColumns);
961 | stringColumns.splice(0, stringColumns.length, ...baseStringColumns);
962 | integerColumns.splice(0, integerColumns.length, ...baseIntegerColumns);
| ^
963 | timeColumns.splice(0, timeColumns.length, ...baseTimeColumns);
964 | timeColumnOptions.splice(0, timeColumnOptions.length, ...baseTimeColumnO
ptions);
|
scubaduck/static/index.html:963:48: SyntaxError: Expected ',', found name
|
961 | stringColumns.splice(0, stringColumns.length, ...baseStringColumns);
962 | integerColumns.splice(0, integerColumns.length, ...baseIntegerColumns);
963 | timeColumns.splice(0, timeColumns.length, ...baseTimeColumns);
| ^
964 | timeColumnOptions.splice(0, timeColumnOptions.length, ...baseTimeColumnO
ptions);
965 | derivedColumns.forEach(d => {
|
scubaduck/static/index.html:963:64: SyntaxError: Expected ',', found ';'
|
961 | stringColumns.splice(0, stringColumns.length, ...baseStringColumns);
962 | integerColumns.splice(0, integerColumns.length, ...baseIntegerColumns);
963 | timeColumns.splice(0, timeColumns.length, ...baseTimeColumns);
| ^
964 | timeColumnOptions.splice(0, timeColumnOptions.length, ...baseTimeColumnO
ptions);
965 | derivedColumns.forEach(d => {
|
scubaduck/static/index.html:964:60: SyntaxError: Expected ',', found name
|
962 | integerColumns.splice(0, integerColumns.length, ...baseIntegerColumns);
963 | timeColumns.splice(0, timeColumns.length, ...baseTimeColumns);
964 | timeColumnOptions.splice(0, timeColumnOptions.length, ...baseTimeColumnO
ptions);
| ^
965 | derivedColumns.forEach(d => {
966 | d.type = d.el.querySelector('.d-type').value;
|
scubaduck/static/index.html:964:82: SyntaxError: Expected ',', found ';'
|
962 | integerColumns.splice(0, integerColumns.length, ...baseIntegerColumns);
963 | timeColumns.splice(0, timeColumns.length, ...baseTimeColumns);
964 | timeColumnOptions.splice(0, timeColumnOptions.length, ...baseTimeColumnO
ptions);
|Ruff is struggling with HTML parsing, so I'll limit it to Python files.
...
1349 | document.getElementById('query_info').textContent =
1350 | `Your query took about ${lastQueryTime} ms`;
|
= help: Remove unnecessary semicolon
scubaduck/static/index.html:1349:54: SyntaxError: Expected an expression
|
1347 | sqlEl.textContent = data.sql;
1348 | view.appendChild(sqlEl);
1349 | document.getElementById('query_info').textContent =
| ^
1350 | `Your query took about ${lastQueryTime} ms`;
1351 | }
|
scubaduck/static/index.html:1350:1: SyntaxError: Unexpected indentation
|
1348 | view.appendChild(sqlEl);
1349 | document.getElementById('query_info').textContent =
1350 | `Your query took about ${lastQueryTime} ms`;
| ^
1351 | }
|
scubaduck/static/index.html:1350:5: SyntaxError: Got unexpected token `
|
1348 | view.appendChild(sqlEl);
1349 | document.getElementById('query_info').textContent =
1350 | `Your query took about ${lastQueryTime} ms`;
| ^
1351 | }
|
scubaduck/static/index.html:1350:11: SyntaxError: Simple statements must be sepa
rated by newlines or semicolons
|
1348 | view.appendChild(sqlEl);
1349 | document.getElementById('query_info').textContent =
1350 | `Your query took about ${lastQueryTime} ms`;
| ^
1351 | }
|
scubaduck/static/index.html:1350:17: SyntaxError: Simple statements must be sepa
rated by newlines or semicolons
|
1348 | view.appendChild(sqlEl);
1349 | document.getElementById('query_info').textContent =
1350 | `Your query took about ${lastQueryTime} ms`;
| ^
1351 | }
|
scubaduck/static/index.html:1350:22: SyntaxError: Simple statements must be sepa
rated by newlines or semicolons
|
1348 | view.appendChild(sqlEl);
1349 | document.getElementById('query_info').textContent =
1350 | `Your query took about ${lastQueryTime} ms`;
| ^
1351 | }
|
scubaduck/static/index.html:1350:28: SyntaxError: Got unexpected token $
|
1348 | view.appendChild(sqlEl);
1349 | document.getElementById('query_info').textContent =
1350 | `Your query took about ${lastQueryTime} ms`;
| ^
1351 | }
|
scubaduck/static/index.html:1350:45: SyntaxError: Simple statements must be sepa
rated by newlines or semicolons
|
1348 | view.appendChild(sqlEl);
1349 | document.getElementById('query_info').textContent =
1350 | `Your query took about ${lastQueryTime} ms`;
| ^
1351 | }
|
scubaduck/static/index.html:1350:47: SyntaxError: Got unexpected token `
|
1348 | view.appendChild(sqlEl);
1349 | document.getElementById('query_info').textContent =
1350 | `Your query took about ${lastQueryTime} ms`;
| ^
1351 | }
|
scubaduck/static/index.html:1350:48: SyntaxError: Expected a statement
|
1348 | view.appendChild(sqlEl);
1349 | document.getElementById('query_info').textContent =
1350 | `Your query took about ${lastQueryTime} ms`;
| ^
1351 | }
|
scubaduck/static/index.html:1350:48: E703 Statement ends with an unnecessary sem
icolon
|
1348 | view.appendChild(sqlEl);
1349 | document.getElementById('query_info').textContent =
1350 | `Your query took about ${lastQueryTime} ms`;
| ^ E703
1351 | }
|
= help: Remove unnecessary semicolon
scubaduck/static/index.html:1350:49: SyntaxError: Expected a statement
|
1348 | view.appendChild(sqlEl);
1349 | document.getElementById('query_info').textContent =
1350 | `Your query took about ${lastQueryTime} ms`;
| ^
1351 | }
1352 |
1353 | function showError(err) {
|
scubaduck/static/index.html:1351:1: SyntaxError: Expected a statement
|
...
1380 | }
| ^
1381 | }
1382 | </script>
|
scubaduck/static/index.html:1380:4: SyntaxError: Expected a statement
|
1378 | select.dispatchEvent(new Event('change'));
1379 | }
1380 | }
| ^
1381 | }
1382 | </script>
1383 | </body>
|
scubaduck/static/index.html:1381:1: SyntaxError: Expected a statement
|
1379 | }
1380 | }
1381 | }
| ^
1382 | </script>
1383 | </body>
|
scubaduck/static/index.html:1381:2: SyntaxError: Expected a statement
|
1379 | }
1380 | }
1381 | }
| ^
1382 | </script>
1383 | </body>
1384 | </html>
|
scubaduck/static/index.html:1382:1: SyntaxError: Expected a statement
|
1380 | }
1381 | }
1382 | </script>
| ^
1383 | </body>
1384 | </html>
|
scubaduck/static/index.html:1382:2: SyntaxError: Expected a statement
|
1380 | }
1381 | }
1382 | </script>
| ^
1383 | </body>
1384 | </html>
|
scubaduck/static/index.html:1382:10: SyntaxError: Expected an expression
|
1380 | }
1381 | }
1382 | </script>
| ^
1383 | </body>
1384 | </html>
|
scubaduck/static/index.html:1383:1: SyntaxError: Expected a statement
|
1381 | }
1382 | </script>
1383 | </body>
| ^
1384 | </html>
|
scubaduck/static/index.html:1383:2: SyntaxError: Expected a statement
|
1381 | }
1382 | </script>
1383 | </body>
| ^
1384 | </html>
|
scubaduck/static/index.html:1383:8: SyntaxError: Expected an expression
|
1381 | }
1382 | </script>
1383 | </body>
| ^
1384 | </html>
|
scubaduck/static/index.html:1384:1: SyntaxError: Expected a statement
|
1382 | </script>
1383 | </body>
1384 | </html>
| ^
|
scubaduck/static/index.html:1384:2: SyntaxError: Expected a statement
|
1382 | </script>
1383 | </body>
1384 | </html>
| ^
|
scubaduck/static/index.html:1384:8: SyntaxError: Expected an expression
|
1382 | </script>
1383 | </body>
1384 | </html>
| ^
|
Found 3230 errors.
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck# ruff check scubaduck/server.
py tests/test_web.py tests/test_server.py >/tmp/ruff.log && tail -n 20 /tmp/ruff
.log
All checks passed!
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck#The Python files passed; now I'll go ahead and run Pyright to check for typing issues.
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck# pyright >/tmp/pyright.log &&
tail -n 20 /tmp/pyright.log(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck# cat /tmp/pyright.log | tail
-n 20
/workspace/scubaduck/scubaduck/server.py
/workspace/scubaduck/scubaduck/server.py:189:28 - error: Argument of type "str
| None" cannot be assigned to parameter "col" of type "str" in function "_time_
expr"
Type "str | None" is not assignable to type "str"
"None" is not assignable to "str" (reportArgumentType)
1 error, 0 warnings, 0 informations
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck#Pyright still complains despite adding a condition. I'll look at line 186 for clues.
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck# sed -n '180,200p' scubaduck/
server.py
return col
def build_query(params: QueryParams, column_types: Dict[str, str] | None = None)
-> str:
select_parts: list[str] = []
group_cols = params.group_by[:]
if params.graph_type == "timeseries":
sec = _granularity_seconds(params.granularity, params.start, params.end)
x_axis = params.x_axis or params.time_column
xexpr = _time_expr(x_axis, column_types, params.time_unit)
if params.start:
bucket_expr = (
f"TIMESTAMP '{params.start}' + INTERVAL '{sec} second' * "
f"CAST(floor((epoch({xexpr}) - epoch(TIMESTAMP '{params.start}')
)/{sec}) AS BIGINT)"
)
else:
bucket_expr = (
f"TIMESTAMP 'epoch' + INTERVAL '{sec} second' * "
f"CAST(floor(epoch({xexpr})/{sec}) AS BIGINT)"
)
select_parts.append(f"{bucket_expr} AS bucket")
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck#I’ll ensure x_axis isn’t None by raising an error if required. Let’s patch.
> *** Begin Patch
> *** Update File: scubaduck/server.py
> @@
> - sec = _granularity_seconds(params.granularity, params.start, params.e
nd)
> - x_axis = params.x_axis or params.time_column
> - xexpr = _time_expr(x_axis, column_types, params.time_unit)
> + sec = _granularity_seconds(params.granularity, params.start, params.e
nd)
> + x_axis = params.x_axis or params.time_column
> + if x_axis is None:
> + raise ValueError("x_axis required for timeseries")
> + xexpr = _time_expr(x_axis, column_types, params.time_unit)
> *** End Patch
> PATCH
Success. Updated the following files:
M scubaduck/server.py(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck# ruff format scubaduck/server
.py tests/test_web.py tests/test_server.py >/tmp/format.log && tail -n 20 /tmp/f
ormat.log
3 files left unchanged(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck# ruff check scubaduck/server.
py tests/test_web.py tests/test_server.py >/tmp/ruff.log && tail -n 20 /tmp/ruff
.log
All checks passed!(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck# pyright >/tmp/pyright.log &&
tail -n 20 /tmp/pyright.log0 errors, 0 warnings, 0 informations(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck# pytest -q >/tmp/pytest.log &
& tail -n 20 /tmp/pytest.logbringing up nodes...
bringing up nodes...
........................................................................ [ 71%]
............................. [100%]
101 passed in 16.62s
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck#All tests are passing, so it's time to make the commit!
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck# git status --short
M scubaduck/server.py
M scubaduck/static/index.html
M tests/test_server.py
M tests/test_web.py
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck#All done with the changes—time to commit them now.
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck# git add scubaduck/server.py
scubaduck/static/index.html tests/test_server.py tests/test_web.py(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck# git commit -m "feat: allow d
isabling time column" && git status --short
[work f11c273] feat: allow disabling time column
4 files changed, 63 insertions(+), 7 deletions(-)(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck# git status --short
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck#I'll open server.py to collect the citations for the new dataclass code.
sed -n '40,64p'
40 show_hits: bool = False
41 x_axis: str | None = None
42 granularity: str = "Auto"
43 fill: str = "0"
44 table: str = "events"
45 time_column: str | None = "timestamp"
46 time_unit: str = "s"
47
48
49 def _load_database(path: Path) -> duckdb.DuckDBPyConnection:
50 if not path.exists():
51 raise FileNotFoundError(path)
52
53 ext = path.suffix.lower()
54 if ext == ".csv":
55 con = duckdb.connect()
56 con.execute(
57 f"CREATE TABLE events AS SELECT * FROM read_csv_auto('{path.
as_posix()}')"
58 )
59 elif ext in {".db", ".sqlite"}:
60 con = duckdb.connect()
61 con.execute("LOAD sqlite")
62 con.execute(f"ATTACH '{path.as_posix()}' AS db (TYPE SQLITE)")
63 tables = [
64 r[0]
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck#I'll find the relevant lines around where build_query starts.
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck# nl -ba scubaduck/server.py |
sed -n '186,214p'
186 if params.graph_type == "timeseries":
187 sec = _granularity_seconds(params.granularity, params.start, par
ams.end)
188 x_axis = params.x_axis or params.time_column
189 if x_axis is None:
190 raise ValueError("x_axis required for timeseries")
191 xexpr = _time_expr(x_axis, column_types, params.time_unit)
192 if params.start:
193 bucket_expr = (
194 f"TIMESTAMP '{params.start}' + INTERVAL '{sec} second' *
"
195 f"CAST(floor((epoch({xexpr}) - epoch(TIMESTAMP '{params.
start}'))/{sec}) AS BIGINT)"
196 )
197 else:
198 bucket_expr = (
199 f"TIMESTAMP 'epoch' + INTERVAL '{sec} second' * "
200 f"CAST(floor(epoch({xexpr})/{sec}) AS BIGINT)"
201 )
202 select_parts.append(f"{bucket_expr} AS bucket")
203 group_cols = ["bucket"] + group_cols
204 has_agg = bool(group_cols) or params.aggregate is not None
205 if has_agg:
206 select_cols = (
207 group_cols[1:] if params.graph_type == "timeseries" else gro
up_cols
208 )
209 select_parts.extend(select_cols)
210 agg = (params.aggregate or "avg").lower()
211
212 def agg_expr(col: str) -> str:
213 expr = col
214 ctype = column_types.get(col, "").upper() if column_types el
se ""(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck# nl -ba scubaduck/server.py |
sed -n '258,280p'
258 lines.append(f"ORDER BY {params.order_by} {params.order_dir}
")
259 elif params.graph_type == "timeseries":
260 lines.append("ORDER BY bucket")
261 if params.limit is not None:
262 lines.append(f"LIMIT {params.limit}")
263 return "\n".join(lines)
264
265 for name, expr in params.derived_columns.items():
266 select_parts.append(f"{expr} AS {name}")
267 select_clause = ", ".join(select_parts) if select_parts else "*"
268 lines = [f"SELECT {select_clause}", f'FROM "{params.table}"']
269 where_parts: list[str] = []
270 if params.time_column:
271 time_expr = _time_expr(params.time_column, column_types, params.
time_unit)
272 else:
273 time_expr = None
274 if time_expr and params.start:
275 where_parts.append(f"{time_expr} >= '{params.start}'")
276 if time_expr and params.end:
277 where_parts.append(f"{time_expr} <= '{params.end}'")
278 for f in params.filters:
279 op = f.op
280 if op in {"empty", "!empty"}:
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck#I need to figure out the line numbers for where the changes start and end.
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck# nl -ba scubaduck/server.py |
sed -n '532,551p'
532 jsonify(
533 {
534 "error": f"Aggregate {agg} cannot be
applied to column {c}",
535 }
536 ),
537 400,
538 )
539 if (params.start is None or params.end is None) and (
540 params.x_axis or params.time_column
541 ):
542 axis = params.x_axis or params.time_column
543 row = cast(
544 tuple[datetime | None, datetime | None],
545 con.execute(
546 f'SELECT min({axis}), max({axis}) FROM "{params.tabl
e}"'
547 ).fetchall()[0],
548 )
549 mn, mx = row
550 divisor = {
551 "s": 1,
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck#I’ll go ahead and patch the code right after line 548.
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck# nl -ba scubaduck/server.py |
sed -n '548,580p'
548 )
549 mn, mx = row
550 divisor = {
551 "s": 1,
552 "ms": 1000,
553 "us": 1_000_000,
554 "ns": 1_000_000_000,
555 }.get(params.time_unit, 1)
556 if isinstance(mn, (int, float)):
557 try:
558 mn = datetime.fromtimestamp(int(mn) / divisor, tz=ti
mezone.utc)
559 except Exception:
560 return (
561 jsonify(
562 {
563 "error": (
564 f"Invalid time value {mn} for column
{axis}"
565 f" with time_unit {params.time_unit}
"
566 )
567 }
568 ),
569 400,
570 )
571 if isinstance(mx, (int, float)):
572 try:
573 mx = datetime.fromtimestamp(int(mx) / divisor, tz=ti
mezone.utc)
574 except Exception:
575 return (
576 jsonify(
577 {
578 "error": (
579 f"Invalid time value {mx} for column
{axis}"
580 f" with time_unit {params.time_unit}
"
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck#I'll focus on the lines where I inserted changes in index.html.
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck# grep -n "(none)" -n scubaduc
k/static/index.html
575: noneOpt.textContent = '(none)';
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck#I'll grab the line ranges, focusing around line 570.
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck# nl -ba scubaduck/static/inde
x.html | sed -n '568,582p'
568 const defOpt = document.createElement('option');
569 defOpt.value = '';
570 defOpt.textContent = '(default)';
571 xAxisSelect.appendChild(defOpt);
572 timeColumnSelect.innerHTML = '';
573 const noneOpt = document.createElement('option');
574 noneOpt.value = '';
575 noneOpt.textContent = '(none)';
576 timeColumnSelect.appendChild(noneOpt);
577 groupsEl.innerHTML = '';
578 allColumns.length = 0;
579 stringColumns.length = 0;
580 integerColumns.length = 0;
581 timeColumns.length = 0;
582 timeColumnOptions.length = 0;
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck#I'll locate the relevant lines for the updateTimeFieldVisibility function and its event handler.
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck# grep -n "updateTimeFieldVisi
bility" -n scubaduck/static/index.html
542:function updateTimeFieldVisibility() {
557:document.getElementById('time_column').addEventListener('change', updateTime
FieldVisibility);
558:updateTimeFieldVisibility();
645: updateTimeFieldVisibility();
1097: updateTimeFieldVisibility();
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck#I'll show around line 540 for now.
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck# nl -ba scubaduck/static/inde
x.html | sed -n '538,560p'
538 }
539 updateSelectedColumns(newType);
540 displayType = newType;
541 }
542 function updateTimeFieldVisibility() {
543 const show = document.getElementById('time_column').value !== '';
544 document.getElementById('start').closest('.field').style.display = sho
w
545 ? 'flex'
546 : 'none';
547 document.getElementById('end').closest('.field').style.display = show
548 ? 'flex'
549 : 'none';
550 }
551 orderDirBtn.addEventListener('click', () => {
552 orderDir = orderDir === 'ASC' ? 'DESC' : 'ASC';
553 updateOrderDirButton();
554 });
555 updateOrderDirButton();
556 graphTypeSel.addEventListener('change', updateDisplayTypeUI);
557 document.getElementById('time_column').addEventListener('change', update
TimeFieldVisibility);
558 updateTimeFieldVisibility();
559
560 function loadColumns(table) {
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck#I'll focus on lines near 1093 after the applyParams function.
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck# nl -ba scubaduck/static/inde
x.html | sed -n '1090,1100p'
1090 const qs = sp.toString();
1091 return qs ? '?' + qs : '';
1092 }
1093
1094 function applyParams(params) {
1095 if (params.table) document.getElementById('table').value = params.tabl
e;
1096 document.getElementById('time_column').value = params.time_column || d
efaultTimeColumn;
1097 updateTimeFieldVisibility();
1098 if (params.time_unit) document.getElementById('time_unit').value = par
ams.time_unit;
1099 document.getElementById('start').value = params.start || '';
1100 document.getElementById('end').value = params.end || '';
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck#After loadColumns, I’m calling updateTimeFieldVisibility to adjust the time field display.
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck# nl -ba scubaduck/static/inde
x.html | sed -n '640,648p'
640 o.textContent = name;
641 timeColumnSelect.appendChild(o);
642 });
643 xAxisSelect.value = '';
644 defaultTimeColumn = guess || timeColumnOptions[0] || '';
645 updateTimeFieldVisibility();
646 Object.keys(groups).forEach(key => {
647 const g = groups[key];
648 const div = document.createElement('div');
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck#I'll get the line ranges for the new tests in the test server and web files.
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck# nl -ba tests/test_server.py
| sed -n '772,806p'
772 assert all(r[2] == r[1] * 2 for r in rows)
773
774
775 def test_default_start_end_returned() -> None:
776 app = server.app
777 client = app.test_client()
778 payload = {
779 "table": "events",
780 "order_by": "timestamp",
781 "limit": 5,
782 "columns": ["timestamp"],
783 }
784 rv = client.post(
785 "/api/query", data=json.dumps(payload), content_type="applicatio
n/json"
786 )
787 data = rv.get_json()
788 assert rv.status_code == 200
789 assert data["start"] == "2024-01-01 00:00:00"
790 assert data["end"] == "2024-01-02 03:00:00"
791
792
793 def test_time_column_none_no_time_filter() -> None:
794 app = server.app
795 client = app.test_client()
796 payload = {
797 "table": "events",
798 "columns": ["timestamp", "event"],
799 "time_column": "",
800 }
801 rv = client.post(
802 "/api/query", data=json.dumps(payload), content_type="applicatio
n/json"
803 )
804 data = rv.get_json()
805 assert rv.status_code == 200
806 assert "start" not in data and "end" not in data(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck# nl -ba tests/test_server.py
| sed -n '806,812p'
806 assert "start" not in data and "end" not in data
807 assert len(data["rows"]) == 4
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck#I'm checking the lines for the test_web file now.
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck# grep -n "time_column_none_hi
des_range" -n tests/test_web.py
121:def test_time_column_none_hides_range(page: Any, server_url: str) -> None:(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck# nl -ba tests/test_web.py | s
ed -n '112,132p'
112 page.goto(server_url)
113 page.wait_for_selector("#time_column option", state="attached")
114 options = page.locator("#time_column option").all_inner_texts()
115 assert "(none)" in options
116 assert "timestamp" in options
117 assert "value" in options
118 assert page.input_value("#time_column") == "timestamp"
119
120
121 def test_time_column_none_hides_range(page: Any, server_url: str) -> Non
e:
122 page.goto(server_url)
123 page.wait_for_selector("#time_column option", state="attached")
124 select_value(page, "#time_column", "")
125 assert page.is_hidden("#start")
126 assert page.is_hidden("#end")
127 page.evaluate("window.lastResults = undefined")
128 page.click("text=Dive")
129 page.wait_for_function("window.lastResults !== undefined")
130 data = page.evaluate("window.lastResults")
131 assert len(data["rows"]) == 4
132 assert "start" not in data and "end" not in data
(scubaduck) root@ee1d42b3cdab:/workspace/scubaduck#Time to finalize the PR with a clear message and use container.make_pr.