Task ID: task_e_682ea3ba6e0c83239104a907d9d69ec7
Building scubaduck @ file:///workspace/scubaduck
⠙ Preparing packages... (0/33)
python-slugify ------------------------------ 9.82 KiB/9.82 KiB
pyee ------------------------------ 14.91 KiB/15.36 KiB
itsdangerous ------------------------------ 14.88 KiB/15.85 KiB
pluggy ------------------------------ 20.06 KiB/20.06 KiB
execnet ------------------------------ 30.88 KiB/39.66 KiB
pytest-xdist ------------------------------ 30.90 KiB/45.03 KiB
requests ------------------------------ 32.00 KiB/63.41 KiB
packaging ------------------------------ 32.00 KiB/64.91 KiB
idna ------------------------------ 64.00 KiB/68.79 KiB
text-unidecode ------------------------------ 76.32 KiB/76.32 KiB
click ------------------------------ 78.88 KiB/99.76 KiB
flask ------------------------------ 46.88 KiB/100.88 KiB
urllib3 ------------------------------ 76.85 KiB/125.66 KiB
jinja2 ------------------------------ 46.88 KiB/131.74 KiB
charset-normalizer ------------------------------ 60.59 KiB/145.08 KiB
certifi ------------------------------ 155.88 KiB/155.88 KiB
werkzeug ------------------------------ 62.88 KiB/219.24 KiB
python-dateutil ------------------------------ 48.00 KiB/224.50 KiB
pytest ------------------------------ 30.88 KiB/335.58 KiB
greenlet ------------------------------ 32.00 KiB/589.71 KiB
pyright ------------------------------ 16.00 KiB/5.31 MiB
ruff ------------------------------ 60.42 KiB/11.02 MiB
Building scubaduck @ file:///workspace/scubaduck
⠙ Preparing packages... (0/33)
python-slugify ------------------------------ 9.82 KiB/9.82 KiB
pyee ------------------------------ 14.91 KiB/15.36 KiB
itsdangerous ------------------------------ 14.88 KiB/15.85 KiB
execnet ------------------------------ 30.88 KiB/39.66 KiB
pytest-xdist ------------------------------ 30.90 KiB/45.03 KiB
requests ------------------------------ 32.00 KiB/63.41 KiB
packaging ------------------------------ 48.00 KiB/64.91 KiB
idna ------------------------------ 68.79 KiB/68.79 KiB
text-unidecode ------------------------------ 76.32 KiB/76.32 KiB
click ------------------------------ 94.88 KiB/99.76 KiB
flask ------------------------------ 62.88 KiB/100.88 KiB
urllib3 ------------------------------ 76.85 KiB/125.66 KiB
jinja2 ------------------------------ 46.88 KiB/131.74 KiB
charset-normalizer ------------------------------ 76.59 KiB/145.08 KiB
certifi ------------------------------ 155.88 KiB/155.88 KiB
werkzeug ------------------------------ 62.88 KiB/219.24 KiB
python-dateutil ------------------------------ 48.00 KiB/224.50 KiB
pytest ------------------------------ 30.88 KiB/335.58 KiB
greenlet ------------------------------ 32.00 KiB/589.71 KiB
pyright ------------------------------ 16.00 KiB/5.31 MiB
ruff ------------------------------ 76.42 KiB/11.02 MiB
duckdb ------------------------------ 272.00 KiB/19.27 MiB
Building scubaduck @ file:///workspace/scubaduck
⠙ Preparing packages... (0/33)
python-slugify ------------------------------ 9.82 KiB/9.82 KiB
pyee ------------------------------ 14.91 KiB/15.36 KiB
itsdangerous ------------------------------ 14.88 KiB/15.85 KiB
execnet ------------------------------ 30.88 KiB/39.66 KiB
pytest-xdist ------------------------------ 30.90 KiB/45.03 KiB
requests ------------------------------ 32.00 KiB/63.41 KiB
packaging ------------------------------ 48.00 KiB/64.91 KiB
idna ------------------------------ 68.79 KiB/68.79 KiB
click ------------------------------ 94.88 KiB/99.76 KiB
flask ------------------------------ 62.88 KiB/100.88 KiB
urllib3 ------------------------------ 76.85 KiB/125.66 KiB
jinja2 ------------------------------ 62.88 KiB/131.74 KiB
charset-normalizer ------------------------------ 108.59 KiB/145.08 KiB
certifi ------------------------------ 155.88 KiB/155.88 KiB
werkzeug ------------------------------ 76.88 KiB/219.24 KiB
python-dateutil ------------------------------ 48.00 KiB/224.50 KiB
pytest ------------------------------ 46.88 KiB/335.58 KiB
greenlet ------------------------------ 32.00 KiB/589.71 KiB
pyright ------------------------------ 16.00 KiB/5.31 MiB
ruff ------------------------------ 92.42 KiB/11.02 MiB
duckdb ------------------------------ 304.00 KiB/19.27 MiB
playwright ------------------------------ 32.00 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠙ Preparing packages... (0/33)
pyee ------------------------------ 14.91 KiB/15.36 KiB
itsdangerous ------------------------------ 15.85 KiB/15.85 KiB
execnet ------------------------------ 30.88 KiB/39.66 KiB
pytest-xdist ------------------------------ 30.90 KiB/45.03 KiB
requests ------------------------------ 48.00 KiB/63.41 KiB
packaging ------------------------------ 48.00 KiB/64.91 KiB
idna ------------------------------ 68.79 KiB/68.79 KiB
click ------------------------------ 94.88 KiB/99.76 KiB
flask ------------------------------ 62.88 KiB/100.88 KiB
urllib3 ------------------------------ 76.85 KiB/125.66 KiB
jinja2 ------------------------------ 62.88 KiB/131.74 KiB
charset-normalizer ------------------------------ 108.59 KiB/145.08 KiB
certifi ------------------------------ 155.88 KiB/155.88 KiB
werkzeug ------------------------------ 76.88 KiB/219.24 KiB
python-dateutil ------------------------------ 48.00 KiB/224.50 KiB
pytest ------------------------------ 46.88 KiB/335.58 KiB
greenlet ------------------------------ 32.00 KiB/589.71 KiB
pyright ------------------------------ 16.00 KiB/5.31 MiB
ruff ------------------------------ 108.42 KiB/11.02 MiB
duckdb ------------------------------ 304.00 KiB/19.27 MiB
playwright ------------------------------ 32.00 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠙ Preparing packages... (0/33)
pyee ------------------------------ 14.91 KiB/15.36 KiB
itsdangerous ------------------------------ 15.85 KiB/15.85 KiB
execnet ------------------------------ 30.88 KiB/39.66 KiB
pytest-xdist ------------------------------ 30.90 KiB/45.03 KiB
requests ------------------------------ 48.00 KiB/63.41 KiB
packaging ------------------------------ 48.00 KiB/64.91 KiB
click ------------------------------ 94.88 KiB/99.76 KiB
flask ------------------------------ 62.88 KiB/100.88 KiB
urllib3 ------------------------------ 76.85 KiB/125.66 KiB
jinja2 ------------------------------ 62.88 KiB/131.74 KiB
charset-normalizer ------------------------------ 108.59 KiB/145.08 KiB
certifi ------------------------------ 155.88 KiB/155.88 KiB
werkzeug ------------------------------ 76.88 KiB/219.24 KiB
python-dateutil ------------------------------ 48.00 KiB/224.50 KiB
pytest ------------------------------ 46.88 KiB/335.58 KiB
greenlet ------------------------------ 32.00 KiB/589.71 KiB
pyright ------------------------------ 16.00 KiB/5.31 MiB
ruff ------------------------------ 124.42 KiB/11.02 MiB
duckdb ------------------------------ 304.00 KiB/19.27 MiB
playwright ------------------------------ 32.00 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠙ Preparing packages... (0/33)
pyee ------------------------------ 14.91 KiB/15.36 KiB
execnet ------------------------------ 30.88 KiB/39.66 KiB
pytest-xdist ------------------------------ 30.90 KiB/45.03 KiB
requests ------------------------------ 48.00 KiB/63.41 KiB
packaging ------------------------------ 48.00 KiB/64.91 KiB
click ------------------------------ 99.76 KiB/99.76 KiB
flask ------------------------------ 62.88 KiB/100.88 KiB
urllib3 ------------------------------ 76.85 KiB/125.66 KiB
jinja2 ------------------------------ 75.67 KiB/131.74 KiB
charset-normalizer ------------------------------ 124.59 KiB/145.08 KiB
certifi ------------------------------ 155.88 KiB/155.88 KiB
werkzeug ------------------------------ 76.88 KiB/219.24 KiB
python-dateutil ------------------------------ 48.00 KiB/224.50 KiB
pytest ------------------------------ 46.88 KiB/335.58 KiB
greenlet ------------------------------ 32.00 KiB/589.71 KiB
pyright ------------------------------ 16.00 KiB/5.31 MiB
ruff ------------------------------ 124.42 KiB/11.02 MiB
duckdb ------------------------------ 320.00 KiB/19.27 MiB
playwright ------------------------------ 32.00 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠙ Preparing packages... (0/33)
pyee ------------------------------ 14.91 KiB/15.36 KiB
execnet ------------------------------ 30.88 KiB/39.66 KiB
pytest-xdist ------------------------------ 30.90 KiB/45.03 KiB
requests ------------------------------ 48.00 KiB/63.41 KiB
packaging ------------------------------ 48.00 KiB/64.91 KiB
click ------------------------------ 99.76 KiB/99.76 KiB
flask ------------------------------ 62.88 KiB/100.88 KiB
urllib3 ------------------------------ 76.85 KiB/125.66 KiB
jinja2 ------------------------------ 75.67 KiB/131.74 KiB
charset-normalizer ------------------------------ 124.59 KiB/145.08 KiB
werkzeug ------------------------------ 76.88 KiB/219.24 KiB
python-dateutil ------------------------------ 48.00 KiB/224.50 KiB
pytest ------------------------------ 62.88 KiB/335.58 KiB
greenlet ------------------------------ 48.00 KiB/589.71 KiB
pyright ------------------------------ 16.00 KiB/5.31 MiB
ruff ------------------------------ 140.42 KiB/11.02 MiB
duckdb ------------------------------ 336.00 KiB/19.27 MiB
playwright ------------------------------ 32.00 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠙ Preparing packages... (0/33)
execnet ------------------------------ 39.66 KiB/39.66 KiB
pytest-xdist ------------------------------ 45.03 KiB/45.03 KiB
requests ------------------------------ 48.00 KiB/63.41 KiB
packaging ------------------------------ 64.91 KiB/64.91 KiB
click ------------------------------ 99.76 KiB/99.76 KiB
flask ------------------------------ 62.88 KiB/100.88 KiB
urllib3 ------------------------------ 76.85 KiB/125.66 KiB
jinja2 ------------------------------ 91.67 KiB/131.74 KiB
charset-normalizer ------------------------------ 140.59 KiB/145.08 KiB
werkzeug ------------------------------ 92.88 KiB/219.24 KiB
python-dateutil ------------------------------ 60.82 KiB/224.50 KiB
pytest ------------------------------ 62.88 KiB/335.58 KiB
greenlet ------------------------------ 48.00 KiB/589.71 KiB
pyright ------------------------------ 16.00 KiB/5.31 MiB
ruff ------------------------------ 172.42 KiB/11.02 MiB
duckdb ------------------------------ 368.00 KiB/19.27 MiB
playwright ------------------------------ 32.00 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠙ Preparing packages... (0/33)
execnet ------------------------------ 39.66 KiB/39.66 KiB
pytest-xdist ------------------------------ 45.03 KiB/45.03 KiB
requests ------------------------------ 63.41 KiB/63.41 KiB
packaging ------------------------------ 64.91 KiB/64.91 KiB
flask ------------------------------ 78.88 KiB/100.88 KiB
urllib3 ------------------------------ 92.85 KiB/125.66 KiB
jinja2 ------------------------------ 91.67 KiB/131.74 KiB
charset-normalizer ------------------------------ 140.59 KiB/145.08 KiB
werkzeug ------------------------------ 92.88 KiB/219.24 KiB
python-dateutil ------------------------------ 76.82 KiB/224.50 KiB
pytest ------------------------------ 77.34 KiB/335.58 KiB
greenlet ------------------------------ 63.91 KiB/589.71 KiB
pyright ------------------------------ 16.00 KiB/5.31 MiB
ruff ------------------------------ 236.42 KiB/11.02 MiB
duckdb ------------------------------ 432.00 KiB/19.27 MiB
playwright ------------------------------ 48.00 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠙ Preparing packages... (0/33)
execnet ------------------------------ 39.66 KiB/39.66 KiB
pytest-xdist ------------------------------ 45.03 KiB/45.03 KiB
requests ------------------------------ 63.41 KiB/63.41 KiB
flask ------------------------------ 78.88 KiB/100.88 KiB
urllib3 ------------------------------ 92.85 KiB/125.66 KiB
jinja2 ------------------------------ 123.67 KiB/131.74 KiB
charset-normalizer ------------------------------ 145.08 KiB/145.08 KiB
werkzeug ------------------------------ 108.88 KiB/219.24 KiB
python-dateutil ------------------------------ 172.82 KiB/224.50 KiB
pytest ------------------------------ 109.34 KiB/335.58 KiB
greenlet ------------------------------ 143.91 KiB/589.71 KiB
pyright ------------------------------ 142.14 KiB/5.31 MiB
ruff ------------------------------ 380.42 KiB/11.02 MiB
duckdb ------------------------------ 576.00 KiB/19.27 MiB
playwright ------------------------------ 48.00 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠙ Preparing packages... (0/33)
execnet ------------------------------ 39.66 KiB/39.66 KiB
pytest-xdist ------------------------------ 45.03 KiB/45.03 KiB
flask ------------------------------ 78.88 KiB/100.88 KiB
urllib3 ------------------------------ 92.85 KiB/125.66 KiB
jinja2 ------------------------------ 123.67 KiB/131.74 KiB
charset-normalizer ------------------------------ 145.08 KiB/145.08 KiB
werkzeug ------------------------------ 108.88 KiB/219.24 KiB
python-dateutil ------------------------------ 188.82 KiB/224.50 KiB
pytest ------------------------------ 109.34 KiB/335.58 KiB
greenlet ------------------------------ 159.91 KiB/589.71 KiB
pyright ------------------------------ 158.14 KiB/5.31 MiB
ruff ------------------------------ 380.42 KiB/11.02 MiB
duckdb ------------------------------ 576.00 KiB/19.27 MiB
playwright ------------------------------ 48.00 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠙ Preparing packages... (0/33)
pytest-xdist ------------------------------ 45.03 KiB/45.03 KiB
flask ------------------------------ 78.88 KiB/100.88 KiB
urllib3 ------------------------------ 92.85 KiB/125.66 KiB
jinja2 ------------------------------ 123.67 KiB/131.74 KiB
charset-normalizer ------------------------------ 145.08 KiB/145.08 KiB
werkzeug ------------------------------ 108.88 KiB/219.24 KiB
python-dateutil ------------------------------ 220.82 KiB/224.50 KiB
pytest ------------------------------ 125.34 KiB/335.58 KiB
greenlet ------------------------------ 191.91 KiB/589.71 KiB
pyright ------------------------------ 190.14 KiB/5.31 MiB
ruff ------------------------------ 412.42 KiB/11.02 MiB
duckdb ------------------------------ 608.00 KiB/19.27 MiB
playwright ------------------------------ 48.00 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠙ Preparing packages... (0/33)
flask ------------------------------ 94.88 KiB/100.88 KiB
urllib3 ------------------------------ 92.85 KiB/125.66 KiB
jinja2 ------------------------------ 131.74 KiB/131.74 KiB
charset-normalizer ------------------------------ 145.08 KiB/145.08 KiB
werkzeug ------------------------------ 108.88 KiB/219.24 KiB
python-dateutil ------------------------------ 220.82 KiB/224.50 KiB
pytest ------------------------------ 125.34 KiB/335.58 KiB
greenlet ------------------------------ 223.91 KiB/589.71 KiB
pyright ------------------------------ 222.14 KiB/5.31 MiB
ruff ------------------------------ 444.42 KiB/11.02 MiB
duckdb ------------------------------ 640.00 KiB/19.27 MiB
playwright ------------------------------ 48.00 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠙ Preparing packages... (0/33)
flask ------------------------------ 100.88 KiB/100.88 KiB
urllib3 ------------------------------ 108.85 KiB/125.66 KiB
jinja2 ------------------------------ 131.74 KiB/131.74 KiB
werkzeug ------------------------------ 124.88 KiB/219.24 KiB
python-dateutil ------------------------------ 224.50 KiB/224.50 KiB
pytest ------------------------------ 141.34 KiB/335.58 KiB
greenlet ------------------------------ 271.91 KiB/589.71 KiB
pyright ------------------------------ 270.14 KiB/5.31 MiB
ruff ------------------------------ 492.42 KiB/11.02 MiB
duckdb ------------------------------ 688.00 KiB/19.27 MiB
playwright ------------------------------ 64.00 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠙ Preparing packages... (0/33)
flask ------------------------------ 100.88 KiB/100.88 KiB
urllib3 ------------------------------ 125.66 KiB/125.66 KiB
jinja2 ------------------------------ 131.74 KiB/131.74 KiB
werkzeug ------------------------------ 140.88 KiB/219.24 KiB
python-dateutil ------------------------------ 224.50 KiB/224.50 KiB
pytest ------------------------------ 173.34 KiB/335.58 KiB
greenlet ------------------------------ 351.91 KiB/589.71 KiB
pyright ------------------------------ 350.14 KiB/5.31 MiB
ruff ------------------------------ 588.42 KiB/11.02 MiB
duckdb ------------------------------ 768.00 KiB/19.27 MiB
playwright ------------------------------ 64.00 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (21/33)
flask ------------------------------ 100.88 KiB/100.88 KiB
urllib3 ------------------------------ 125.66 KiB/125.66 KiB
jinja2 ------------------------------ 131.74 KiB/131.74 KiB
werkzeug ------------------------------ 140.88 KiB/219.24 KiB
pytest ------------------------------ 173.34 KiB/335.58 KiB
greenlet ------------------------------ 383.91 KiB/589.71 KiB
pyright ------------------------------ 382.14 KiB/5.31 MiB
ruff ------------------------------ 604.42 KiB/11.02 MiB
duckdb ------------------------------ 800.00 KiB/19.27 MiB
playwright ------------------------------ 64.00 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (21/33)
flask ------------------------------ 100.88 KiB/100.88 KiB
urllib3 ------------------------------ 125.66 KiB/125.66 KiB
werkzeug ------------------------------ 140.88 KiB/219.24 KiB
pytest ------------------------------ 173.34 KiB/335.58 KiB
greenlet ------------------------------ 415.91 KiB/589.71 KiB
pyright ------------------------------ 414.14 KiB/5.31 MiB
ruff ------------------------------ 636.42 KiB/11.02 MiB
duckdb ------------------------------ 832.00 KiB/19.27 MiB
playwright ------------------------------ 64.00 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (21/33)
urllib3 ------------------------------ 125.66 KiB/125.66 KiB
werkzeug ------------------------------ 140.88 KiB/219.24 KiB
pytest ------------------------------ 189.34 KiB/335.58 KiB
greenlet ------------------------------ 415.91 KiB/589.71 KiB
pyright ------------------------------ 414.14 KiB/5.31 MiB
ruff ------------------------------ 636.42 KiB/11.02 MiB
duckdb ------------------------------ 832.00 KiB/19.27 MiB
playwright ------------------------------ 64.00 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (21/33)
werkzeug ------------------------------ 140.88 KiB/219.24 KiB
pytest ------------------------------ 189.34 KiB/335.58 KiB
greenlet ------------------------------ 463.91 KiB/589.71 KiB
pyright ------------------------------ 574.14 KiB/5.31 MiB
ruff ------------------------------ 812.42 KiB/11.02 MiB
duckdb ------------------------------ 992.00 KiB/19.27 MiB
playwright ------------------------------ 96.00 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (21/33)
pytest ------------------------------ 285.34 KiB/335.58 KiB
greenlet ------------------------------ 495.91 KiB/589.71 KiB
pyright ------------------------------ 1.21 MiB/5.31 MiB
ruff ------------------------------ 1.42 MiB/11.02 MiB
duckdb ------------------------------ 1.64 MiB/19.27 MiB
playwright ------------------------------ 256.00 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (21/33)
pytest ------------------------------ 285.34 KiB/335.58 KiB
greenlet ------------------------------ 495.91 KiB/589.71 KiB
pyright ------------------------------ 1.24 MiB/5.31 MiB
ruff ------------------------------ 1.47 MiB/11.02 MiB
duckdb ------------------------------ 1.67 MiB/19.27 MiB
playwright ------------------------------ 272.00 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (21/33)
greenlet ------------------------------ 543.91 KiB/589.71 KiB
pyright ------------------------------ 1.58 MiB/5.31 MiB
ruff ------------------------------ 1.95 MiB/11.02 MiB
duckdb ------------------------------ 2.15 MiB/19.27 MiB
playwright ------------------------------ 763.00 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (21/33)
greenlet ------------------------------ 589.71 KiB/589.71 KiB
pyright ------------------------------ 1.79 MiB/5.31 MiB
ruff ------------------------------ 2.74 MiB/11.02 MiB
duckdb ------------------------------ 3.00 MiB/19.27 MiB
playwright ------------------------------ 1.56 MiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (21/33)
pyright ------------------------------ 1.81 MiB/5.31 MiB
ruff ------------------------------ 2.83 MiB/11.02 MiB
duckdb ------------------------------ 3.06 MiB/19.27 MiB
playwright ------------------------------ 1.62 MiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (21/33)
pyright ------------------------------ 1.84 MiB/5.31 MiB
ruff ------------------------------ 4.20 MiB/11.02 MiB
duckdb ------------------------------ 4.41 MiB/19.27 MiB
playwright ------------------------------ 2.95 MiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (21/33)
pyright ------------------------------ 1.92 MiB/5.31 MiB
ruff ------------------------------ 5.56 MiB/11.02 MiB
duckdb ------------------------------ 5.80 MiB/19.27 MiB
playwright ------------------------------ 4.33 MiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠸ Preparing packages... (28/33)
pyright ------------------------------ 1.99 MiB/5.31 MiB
ruff ------------------------------ 7.16 MiB/11.02 MiB
duckdb ------------------------------ 7.44 MiB/19.27 MiB
playwright ------------------------------ 5.96 MiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠸ Preparing packages... (28/33)
pyright ------------------------------ 2.08 MiB/5.31 MiB
ruff ------------------------------ 8.72 MiB/11.02 MiB
duckdb ------------------------------ 9.06 MiB/19.27 MiB
playwright ------------------------------ 7.57 MiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠸ Preparing packages... (28/33)
pyright ------------------------------ 2.12 MiB/5.31 MiB
ruff ------------------------------ 10.14 MiB/11.02 MiB
duckdb ------------------------------ 10.48 MiB/19.27 MiB
playwright ------------------------------ 8.95 MiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠸ Preparing packages... (28/33)
pyright ------------------------------ 2.14 MiB/5.31 MiB
duckdb ------------------------------ 11.42 MiB/19.27 MiB
playwright ------------------------------ 9.93 MiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠸ Preparing packages... (28/33)
pyright ------------------------------ 2.17 MiB/5.31 MiB
duckdb ------------------------------ 12.33 MiB/19.27 MiB
playwright ------------------------------ 10.76 MiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠼ Preparing packages... (29/33)
pyright ------------------------------ 2.20 MiB/5.31 MiB
duckdb ------------------------------ 14.75 MiB/19.27 MiB
playwright ------------------------------ 13.20 MiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠼ Preparing packages... (29/33)
pyright ------------------------------ 2.25 MiB/5.31 MiB
duckdb ------------------------------ 17.11 MiB/19.27 MiB
playwright ------------------------------ 15.62 MiB/43.05 MiB
Built scubaduck @ file:///workspace/scubaduck
⠼ Preparing packages... (29/33)
pyright ------------------------------ 2.27 MiB/5.31 MiB
duckdb ------------------------------ 17.72 MiB/19.27 MiB
playwright ------------------------------ 16.19 MiB/43.05 MiB
⠼ Preparing packages... (29/33)
pyright ------------------------------ 2.31 MiB/5.31 MiB
duckdb ------------------------------ 18.95 MiB/19.27 MiB
playwright ------------------------------ 17.37 MiB/43.05 MiB
⠼ Preparing packages... (29/33)
pyright ------------------------------ 2.39 MiB/5.31 MiB
playwright ------------------------------ 19.05 MiB/43.05 MiB
⠼ Preparing packages... (29/33)
pyright ------------------------------ 2.41 MiB/5.31 MiB
playwright ------------------------------ 20.53 MiB/43.05 MiB
⠴ Preparing packages... (31/33)
pyright ------------------------------ 2.47 MiB/5.31 MiB
playwright ------------------------------ 24.67 MiB/43.05 MiB
⠴ Preparing packages... (31/33)
pyright ------------------------------ 2.64 MiB/5.31 MiB
playwright ------------------------------ 26.20 MiB/43.05 MiB
⠴ Preparing packages... (31/33)
pyright ------------------------------ 2.78 MiB/5.31 MiB
playwright ------------------------------ 28.02 MiB/43.05 MiB
⠴ Preparing packages... (31/33)
pyright ------------------------------ 2.94 MiB/5.31 MiB
playwright ------------------------------ 29.66 MiB/43.05 MiB
⠦ Preparing packages... (31/33)
pyright ------------------------------ 3.11 MiB/5.31 MiB
playwright ------------------------------ 31.57 MiB/43.05 MiB
⠦ Preparing packages... (31/33)
pyright ------------------------------ 3.23 MiB/5.31 MiB
playwright ------------------------------ 33.34 MiB/43.05 MiB
⠦ Preparing packages... (31/33)
pyright ------------------------------ 3.37 MiB/5.31 MiB
playwright ------------------------------ 34.86 MiB/43.05 MiB
⠦ Preparing packages... (31/33)
pyright ------------------------------ 3.53 MiB/5.31 MiB
playwright ------------------------------ 36.89 MiB/43.05 MiB
⠧ Preparing packages... (31/33)
pyright ------------------------------ 3.78 MiB/5.31 MiB
playwright ------------------------------ 38.38 MiB/43.05 MiB
⠧ Preparing packages... (31/33)
pyright ------------------------------ 3.95 MiB/5.31 MiB
playwright ------------------------------ 40.22 MiB/43.05 MiB
⠧ Preparing packages... (31/33)
pyright ------------------------------ 4.23 MiB/5.31 MiB
playwright ------------------------------ 41.40 MiB/43.05 MiB
⠧ Preparing packages... (31/33)
pyright ------------------------------ 4.37 MiB/5.31 MiB
⠇ Preparing packages... (32/33)
pyright ------------------------------ 4.41 MiB/5.31 MiB
⠇ Preparing packages... (32/33)
⠇ (33/33)
Prepared 33 packages in 1.45s
░░░░░░░░░░░░░░░░░░░░ [0/0] Installing wheels...
░░░░░░░░░░░░░░░░░░░░ [0/33] Installing wheels...
░░░░░░░░░░░░░░░░░░░░ [0/33] pluggy==1.6.0
░░░░░░░░░░░░░░░░░░░░ [0/33] packaging==25.0
█░░░░░░░░░░░░░░░░░░░ [2/33] pytest-base-url==2.1.0
█░░░░░░░░░░░░░░░░░░░ [3/33] pytest-base-url==2.1.0
█░░░░░░░░░░░░░░░░░░░ [3/33] pytest-base-url==2.1.0
█░░░░░░░░░░░░░░░░░░░ [3/33] pytest-base-url==2.1.0
█░░░░░░░░░░░░░░░░░░░ [3/33] iniconfig==2.1.0
██░░░░░░░░░░░░░░░░░░ [4/33] iniconfig==2.1.0
██░░░░░░░░░░░░░░░░░░ [4/33] requests==2.32.3
███░░░░░░░░░░░░░░░░░ [5/33] requests==2.32.3
███░░░░░░░░░░░░░░░░░ [5/33] urllib3==2.4.0
███░░░░░░░░░░░░░░░░░ [6/33] urllib3==2.4.0
███░░░░░░░░░░░░░░░░░ [6/33] blinker==1.9.0
████░░░░░░░░░░░░░░░░ [7/33] blinker==1.9.0
████░░░░░░░░░░░░░░░░ [7/33] execnet==2.1.1
████░░░░░░░░░░░░░░░░ [8/33] execnet==2.1.1
████░░░░░░░░░░░░░░░░ [8/33] six==1.17.0
█████░░░░░░░░░░░░░░░ [9/33] six==1.17.0
███████████████████░ [32/33] pyright==1.1.400
Installed 33 packages in 80ms
+ blinker==1.9.0
+ certifi==2025.4.26
+ charset-normalizer==3.4.2
+ click==8.2.0
+ duckdb==1.2.2
+ execnet==2.1.1
+ flask==3.1.1
+ greenlet==3.2.2
+ idna==3.10
+ iniconfig==2.1.0
+ itsdangerous==2.2.0
+ jinja2==3.1.6
+ markupsafe==3.0.2
+ nodeenv==1.9.1
+ packaging==25.0
+ playwright==1.52.0
+ pluggy==1.6.0
+ pyee==13.0.0
+ pyright==1.1.400
+ pytest==8.3.5
+ pytest-base-url==2.1.0
+ pytest-playwright==0.7.0
+ pytest-xdist==3.6.1
+ python-dateutil==2.9.0.post0
+ python-slugify==8.0.4
+ requests==2.32.3
+ ruff==0.11.10
+ scubaduck==0.1.0 (from file:///workspace/scubaduck)
+ six==1.17.0
+ text-unidecode==1.3
+ typing-extensions==4.13.2
+ urllib3==2.4.0
+ werkzeug==3.1.3
++ source .venv/bin/activate
+++ '[' -z '' ']'
+++ '[' -n x ']'
+++ SCRIPT_PATH=.venv/bin/activate
+++ '[' .venv/bin/activate = /tmp/VGcgkW-setup_script.sh ']'
+++ deactivate nondestructive
+++ unset -f pydoc
+++ '[' -z '' ']'
+++ '[' -z '' ']'
+++ hash -r
+++ '[' -z '' ']'
+++ unset VIRTUAL_ENV
+++ unset VIRTUAL_ENV_PROMPT
+++ '[' '!' nondestructive = nondestructive ']'
+++ VIRTUAL_ENV=/workspace/scubaduck/.venv
+++ '[' linux-gnu = cygwin ']'
+++ '[' linux-gnu = msys ']'
+++ export VIRTUAL_ENV
+++ '[' -z '' ']'
+++ unset SCRIPT_PATH
+++ _OLD_VIRTUAL_PATH=/root/.cargo/bin:/root/.rbenv/shims:/root/.rbenv/bin:/root/.rbenv/shims:/root/.local/share/swiftly/bin:/root/.bun/bin:/root/.nvm/versions/node/v22.15.1/bin:/root/.pyenv/shims:3441PYENV_ROOT/shims:/root/.pyenv/bin:/usr/local/go/bin:/root/go/bin:/root/.rbenv/bin:/root/.rbenv/shims:/root/.bun/bin:/root/.local/bin:/root/.pyenv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+++ PATH=/workspace/scubaduck/.venv/bin:/root/.cargo/bin:/root/.rbenv/shims:/root/.rbenv/bin:/root/.rbenv/shims:/root/.local/share/swiftly/bin:/root/.bun/bin:/root/.nvm/versions/node/v22.15.1/bin:/root/.pyenv/shims:3441PYENV_ROOT/shims:/root/.pyenv/bin:/usr/local/go/bin:/root/go/bin:/root/.rbenv/bin:/root/.rbenv/shims:/root/.bun/bin:/root/.local/bin:/root/.pyenv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+++ export PATH
+++ '[' xscubaduck '!=' x ']'
+++ VIRTUAL_ENV_PROMPT='(scubaduck) '
+++ export VIRTUAL_ENV_PROMPT
+++ '[' -z '' ']'
+++ '[' -z '' ']'
+++ _OLD_VIRTUAL_PS1=
+++ PS1='(scubaduck) '
+++ export PS1
+++ alias pydoc
+++ true
+++ hash -r
++ playwright install chromium
Downloading Chromium 136.0.7103.25 (playwright build v1169) from https://cdn.playwright.dev/dbazure/download/playwright/builds/chromium/1169/chromium-linux.zip
167.7 MiB [] 0% 0.0s167.7 MiB [] 0% 24.1s167.7 MiB [] 0% 32.4s167.7 MiB [] 0% 31.6s167.7 MiB [] 0% 23.7s167.7 MiB [] 0% 17.5s167.7 MiB [] 0% 11.1s167.7 MiB [] 1% 10.6s167.7 MiB [] 1% 10.4s167.7 MiB [] 1% 9.5s167.7 MiB [] 1% 9.2s167.7 MiB [] 2% 8.7s167.7 MiB [] 3% 5.8s167.7 MiB [] 4% 4.5s167.7 MiB [] 5% 4.0s167.7 MiB [] 6% 3.7s167.7 MiB [] 8% 3.1s167.7 MiB [] 9% 2.9s167.7 MiB [] 10% 2.7s167.7 MiB [] 12% 2.3s167.7 MiB [] 14% 2.1s167.7 MiB [] 15% 2.0s167.7 MiB [] 16% 2.0s167.7 MiB [] 18% 1.8s167.7 MiB [] 19% 1.7s167.7 MiB [] 21% 1.6s167.7 MiB [] 23% 1.4s167.7 MiB [] 25% 1.4s167.7 MiB [] 26% 1.3s167.7 MiB [] 28% 1.3s167.7 MiB [] 29% 1.2s167.7 MiB [] 31% 1.2s167.7 MiB [] 33% 1.1s167.7 MiB [] 35% 1.0s167.7 MiB [] 37% 1.0s167.7 MiB [] 39% 0.9s167.7 MiB [] 40% 0.9s167.7 MiB [] 42% 0.9s167.7 MiB [] 44% 0.8s167.7 MiB [] 45% 0.8s167.7 MiB [] 47% 0.8s167.7 MiB [] 49% 0.7s167.7 MiB [] 52% 0.7s167.7 MiB [] 53% 0.7s167.7 MiB [] 54% 0.6s167.7 MiB [] 54% 0.7s167.7 MiB [] 56% 0.6s167.7 MiB [] 56% 0.7s167.7 MiB [] 57% 0.7s167.7 MiB [] 58% 0.6s167.7 MiB [] 59% 0.6s167.7 MiB [] 61% 0.6s167.7 MiB [] 62% 0.6s167.7 MiB [] 64% 0.6s167.7 MiB [] 65% 0.6s167.7 MiB [] 67% 0.5s167.7 MiB [] 68% 0.5s167.7 MiB [] 69% 0.5s167.7 MiB [] 71% 0.5s167.7 MiB [] 72% 0.4s167.7 MiB [] 73% 0.4s167.7 MiB [] 75% 0.4s167.7 MiB [] 77% 0.4s167.7 MiB [] 78% 0.3s167.7 MiB [] 80% 0.3s167.7 MiB [] 81% 0.3s167.7 MiB [] 83% 0.3s167.7 MiB [] 84% 0.2s167.7 MiB [] 86% 0.2s167.7 MiB [] 88% 0.2s167.7 MiB [] 89% 0.2s167.7 MiB [] 91% 0.1s167.7 MiB [] 93% 0.1s167.7 MiB [] 94% 0.1s167.7 MiB [] 95% 0.1s167.7 MiB [] 97% 0.0s167.7 MiB [] 99% 0.0s167.7 MiB [] 100% 0.0s
Chromium 136.0.7103.25 (playwright build v1169) downloaded to /root/.cache/ms-playwright/chromium-1169
Downloading FFMPEG playwright build v1011 from https://cdn.playwright.dev/dbazure/download/playwright/builds/ffmpeg/1011/ffmpeg-linux.zip
2.3 MiB [] 0% 0.0s2.3 MiB [] 5% 0.4s2.3 MiB [] 15% 0.2s2.3 MiB [] 35% 0.1s2.3 MiB [] 76% 0.0s2.3 MiB [] 100% 0.0s
FFMPEG playwright build v1011 downloaded to /root/.cache/ms-playwright/ffmpeg-1011
Downloading Chromium Headless Shell 136.0.7103.25 (playwright build v1169) from https://cdn.playwright.dev/dbazure/download/playwright/builds/chromium/1169/chromium-headless-shell-linux.zip
101.4 MiB [] 0% 0.0s101.4 MiB [] 0% 14.6s101.4 MiB [] 0% 8.5s101.4 MiB [] 1% 4.8s101.4 MiB [] 2% 2.5s101.4 MiB [] 4% 1.7s101.4 MiB [] 7% 1.3s101.4 MiB [] 9% 1.1s101.4 MiB [] 12% 1.0s101.4 MiB [] 15% 0.8s101.4 MiB [] 17% 0.8s101.4 MiB [] 21% 0.7s101.4 MiB [] 24% 0.6s101.4 MiB [] 29% 0.5s101.4 MiB [] 33% 0.5s101.4 MiB [] 36% 0.5s101.4 MiB [] 39% 0.4s101.4 MiB [] 44% 0.4s101.4 MiB [] 47% 0.3s101.4 MiB [] 49% 0.3s101.4 MiB [] 53% 0.3s101.4 MiB [] 54% 0.3s101.4 MiB [] 58% 0.3s101.4 MiB [] 61% 0.2s101.4 MiB [] 63% 0.2s101.4 MiB [] 66% 0.2s101.4 MiB [] 69% 0.2s101.4 MiB [] 74% 0.2s101.4 MiB [] 77% 0.1s101.4 MiB [] 82% 0.1s101.4 MiB [] 85% 0.1s101.4 MiB [] 89% 0.1s101.4 MiB [] 91% 0.1s101.4 MiB [] 94% 0.0s101.4 MiB [] 96% 0.0s101.4 MiB [] 99% 0.0s101.4 MiB [] 100% 0.0s
Chromium Headless Shell 136.0.7103.25 (playwright build v1169) downloaded to /root/.cache/ms-playwright/chromium_headless_shell-1169
++ echo 'source .venv/bin/activate'Just went through the AGENTS.md file in the repo's root directory.
lslsls -a
bash-5.2#
bash-5.2# ls -a
. .. scubaduckbash-5.2# cd scubaduck
bash-5.2# ls
AGENTS.md codex_setup.sh scubaduck tests
README.md pyproject.toml stubs uv.lockbash-5.2# cat AGENTS.md
- Use "source .venv/bin/activate" to activate the virtual environment before
running any commands
- ONLY use non-interactive commands like cat, sed, apply_patch to do edits.
Do NOT use interactive editors.
- Do NOT attempt to install packages. Only the packages specified in
pyproject.toml are available. You cannot add new packages. If you
desperately want another package, make a note of it in the final PR
description.
- Use conventional commits to format PR title
- There are no nested AGENTS.md files, this is the only agents file
- When using Playwright, ONLY use chromium browser. This browser is already
installed.
- Use "ruff check" to check lint, "ruff format" to autoformat files and
"pyright" to typecheck.
- When writing the PR description, include the original prompt supplied by the
user.
- When you add functionality to the server, add server tests to
tests/test_server.py. When you add features to the frontend, add frontend
tests to tests/test_web.py
bash-5.2#I see some rules like using ruff check, avoiding package installs, and adding tests for server/frontend features. I'll search the repo to locate the relevant code for the Time Column update.
bash-5.2# ls -R | head
.:
AGENTS.md
README.md
codex_setup.sh
pyproject.toml
scubaduck
stubs
tests
uv.lockbash-5.2# ls scubaduck
__init__.py sample.csv server.py staticbash-5.2# ls scubaduck/static
index.html jsbash-5.2# ls scubaduck/static/js
chip_input.js timeseries_chart.jsbash-5.2# grep -n "Time" -r scubaduck/static
scubaduck/static/js/timeseries_chart.js:4:function showTimeSeries(data) {
scubaduck/static/js/timeseries_chart.js:7: return new Date(s).getTime();
scubaduck/static/js/timeseries_chart.js:9: return new Date(s + 'Z').getTime()
;
scubaduck/static/js/timeseries_chart.js:199: t = d.getTime() - adj * 8640
0000;
scubaduck/static/index.html:184: <div id="header">sample.csv - <select id="tabl
e"></select> <select id="graph_type"><option value="samples">Samples</option><op
tion value="table">Table</option><option value="timeseries">Time Series</option>
</select></div>
scubaduck/static/index.html:194: <label>Time Column</label>
scubaduck/static/index.html:349:const baseTimeColumns = [];
scubaduck/static/index.html:351:const baseTimeColumnOptions = [];
scubaduck/static/index.html:356:let defaultTimeColumn = '';
scubaduck/static/index.html:497: if (isTimeColumn(cb.value) || isStringColu
mn(cb.value)) {
scubaduck/static/index.html:531: baseTimeColumns.length = 0;
scubaduck/static/index.html:532: baseTimeColumnOptions.length = 0;
scubaduck/static/index.html:535: time: {name: 'Time', cols: [], ul: null},
scubaduck/static/index.html:548: const isTimeType = t.includes('TIMESTAMP')
|| t.includes('DATE') || t.includes('TIME');
scubaduck/static/index.html:549: if (isNumeric || isTimeType) {
scubaduck/static/index.html:551: baseTimeColumnOptions.push(c.name);
scubaduck/static/index.html:554: if (isTimeType) {
scubaduck/static/index.html:557: baseTimeColumns.push(c.name);
scubaduck/static/index.html:588: defaultTimeColumn = guess || timeColumnOptio
ns[0] || '';
scubaduck/static/index.html:746:function isTimeColumn(name) {
scubaduck/static/index.html:902: timeColumns.splice(0, timeColumns.length, ...b
aseTimeColumns);
scubaduck/static/index.html:903: timeColumnOptions.splice(0, timeColumnOptions.
length, ...baseTimeColumnOptions);
scubaduck/static/index.html:921:let lastQueryTime = 0;
scubaduck/static/index.html:946: lastQueryTime = Math.round(performance.now
() - queryStart);
scubaduck/static/index.html:1027: document.getElementById('time_column').value
= params.time_column || defaultTimeColumn;
scubaduck/static/index.html:1172: if (isTimeColumn(col)) {
scubaduck/static/index.html:1242: showTimeSeries(data);
scubaduck/static/index.html:1261: `Your query took about ${lastQueryTime} ms`
;bash-5.2# grep -n "time" -r scubaduck/server.py
7:from datetime import datetime, timedelta, timezone
9:import time
46: time_column: str = "timestamp"
128:def parse_time(val: str | None) -> str | None:
129: """Parse an absolute or relative time string into ``YYYY-MM-DD HH:MM:SS`
`."""
134: dt = datetime.now(timezone.utc)
135: return dt.replace(microsecond=0).strftime("%Y-%m-%d %H:%M:%S")
141: delta: timedelta
143: delta = timedelta(hours=qty)
145: delta = timedelta(days=qty)
147: delta = timedelta(weeks=qty)
149: delta = timedelta(weeks=2 * qty)
152: dt = datetime.now(timezone.utc) + delta
153: return dt.replace(microsecond=0).strftime("%Y-%m-%d %H:%M:%S")
156: return dt.replace(microsecond=0, tzinfo=None).strftime("%Y-%m-%d %H:%M:%
S")
193:def _time_expr(col: str, column_types: Dict[str, str] | None) -> str:
194: """Return SQL expression for column interpreted as timestamp."""
218: if params.graph_type == "timeseries":
220: x_axis = params.x_axis or params.time_column
221: xexpr = _time_expr(x_axis, column_types)
237: group_cols[1:] if params.graph_type == "timeseries" else group_c
ols
286: elif params.graph_type == "timeseries":
297: time_expr = _time_expr(params.time_column, column_types)
299: where_parts.append(f"{time_expr} >= '{params.start}'")
301: where_parts.append(f"{time_expr} <= '{params.end}'")
334: elif params.graph_type == "timeseries":
393: if time.time() - ts > CACHE_TTL:
396: sample_cache[key] = (vals, time.time())
400: sample_cache[key] = (vals, time.time())
432: start = parse_time(payload.get("start"))
433: end = parse_time(payload.get("end"))
453: time_column=payload.get("time_column", "timestamp"),
463: if params.time_column not in column_types:
464: return jsonify({"error": "Invalid time_column"}), 400
466: if params.graph_type not in {"table", "timeseries"} and (
472: "error": "group_by, aggregate and show_hits are only
valid for table or timeseries view"
480: if params.graph_type == "timeseries":
482: params.x_axis = params.time_column
486: is_time = any(t in ctype for t in ["TIMESTAMP", "DATE", "TIME"])
499: if not (is_time or is_numeric):
500: return jsonify({"error": "x_axis must be a time column"}), 4
00
510: if params.group_by or params.graph_type == "timeseries":
514: allow_time = False
517: allow_time = True
520: allow_time = False
521: if need_numeric or allow_time:
540: is_time = "TIMESTAMP" in ctype or "DATE" in ctype or "TI
ME" in ctype
550: if allow_time and not (is_numeric or is_time):
560: axis = params.x_axis or params.time_column
562: tuple[datetime | None, datetime | None],
569: mn = datetime.fromtimestamp(int(mn), tz=timezone.utc)
571: mx = datetime.fromtimestamp(int(mx), tz=timezone.utc)
574: mn.strftime("%Y-%m-%d %H:%M:%S") if not isinstance(mn, s
tr) else mn
578: mx.strftime("%Y-%m-%d %H:%M:%S") if not isinstance(mx, s
tr) else mx
582: if params.graph_type == "timeseries":
bash-5.2#I need to view more lines around _time_expr to get all the context.
bash-5.2# sed -n '182,210p' scubaduck/server.py
try:
s = dtparser.parse(start)
e = dtparser.parse(end)
except Exception:
return 3600
total = max((e - s).total_seconds(), 1)
buckets = 100 if gran == "auto" else 500
return max(int(total // buckets), 1)
return 3600
def _time_expr(col: str, column_types: Dict[str, str] | None) -> str:
"""Return SQL expression for column interpreted as timestamp."""
if column_types is None:
return col
ctype = column_types.get(col, "").upper()
if not any(t in ctype for t in ["TIMESTAMP", "DATE", "TIME"]):
if any(
t in ctype
for t in [
"INT",
"DECIMAL",
"REAL",
"DOUBLE",
"FLOAT",
"NUMERIC",
"HUGEINT",
]
):bash-5.2# sed -n '210,280p' scubaduck/server.py
):
return f"TIMESTAMP 'epoch' + INTERVAL '1 second' * {col}"
return col
def build_query(params: QueryParams, column_types: Dict[str, str] | None = None)
-> str:
select_parts: list[str] = []
group_cols = params.group_by[:]
if params.graph_type == "timeseries":
sec = _granularity_seconds(params.granularity, params.start, params.end)
x_axis = params.x_axis or params.time_column
xexpr = _time_expr(x_axis, column_types)
if params.start:
bucket_expr = (
f"TIMESTAMP '{params.start}' + INTERVAL '{sec} second' * "
f"CAST(floor((epoch({xexpr}) - epoch(TIMESTAMP '{params.start}')
)/{sec}) AS BIGINT)"
)
else:
bucket_expr = (
f"TIMESTAMP 'epoch' + INTERVAL '{sec} second' * "
f"CAST(floor(epoch({xexpr})/{sec}) AS BIGINT)"
)
select_parts.append(f"{bucket_expr} AS bucket")
group_cols = ["bucket"] + group_cols
has_agg = bool(group_cols) or params.aggregate is not None
if has_agg:
select_cols = (
group_cols[1:] if params.graph_type == "timeseries" else group_cols
)
select_parts.extend(select_cols)
agg = (params.aggregate or "avg").lower()
def agg_expr(col: str) -> str:
if agg.startswith("p"):
quant = float(agg[1:]) / 100
return f"quantile({col}, {quant})"
if agg == "count distinct":
return f"count(DISTINCT {col})"
if agg == "avg" and column_types is not None:
ctype = column_types.get(col, "").upper()
if "TIMESTAMP" in ctype or "DATE" in ctype or "TIME" in ctype:
return (
"TIMESTAMP 'epoch' + INTERVAL '1 second' * "
f"CAST(avg(epoch({col})) AS BIGINT)"
)
return f"{agg}({col})"
for col in params.columns:
if col in group_cols:
continue
select_parts.append(f"{agg_expr(col)} AS {col}")
if params.show_hits:
select_parts.insert(len(group_cols), "count(*) AS Hits")
else:
select_parts.extend(params.columns)
if has_agg and params.derived_columns:
inner_params = replace(
params,
derived_columns={},
order_by=None,
limit=None,
)
inner_sql = build_query(inner_params, column_types)
outer_select = ["t.*"] + [
f"{expr} AS {name}" for name, expr in params.derived_columns.items()
]
indented_inner = "\n".join(" " + line for line in inner_sql.splitline
s())
lines = [
f"SELECT {', '.join(outer_select)}",
"FROM (",bash-5.2# grep -n "_time_expr" -n scubaduck/server.py
193:def _time_expr(col: str, column_types: Dict[str, str] | None) -> str:
221: xexpr = _time_expr(x_axis, column_types)
297: time_expr = _time_expr(params.time_column, column_types)
bash-5.2#I'll take a look at the HTML around the Time Column to understand its structure.
bash-5.2# grep -n "time_column" -n scubaduck/static/index.html
195: <select id="time_column"></select>
518: const timeColumnSelect = document.getElementById('time_column');
749: const sel = document.getElementById('time_column').value;
958: time_column: document.getElementById('time_column').value,
1001: if (params.time_column) sp.set('time_column', params.time_column);
1027: document.getElementById('time_column').value = params.time_column || defa
ultTimeColumn;
1086: if (sp.has('time_column')) params.time_column = sp.get('time_column');
bash-5.2#I'll grab the lines around 190 to focus on relevant changes.
bash-5.2# sed -n '180,210p' scubaduck/static/index.html
/* Column resizer removed */
</style>
</head>
<body>
<div id="header">sample.csv - <select id="table"></select> <select id="graph_t
ype"><option value="samples">Samples</option><option value="table">Table</option
><option value="timeseries">Time Series</option></select></div>
<div id="content">
<div id="sidebar">
<div id="tabs">
<button class="tab active" data-tab="settings">View Settings</button>
<button class="tab" data-tab="columns">Columns</button>
<button id="dive" onclick="dive()">Dive</button>
</div>
<div id="settings" class="tab-content active">
<div class="field">
<label>Time Column</label>
<select id="time_column"></select>
</div>
<div class="field">
<label>Start<span class="help" title="Sets the start/end of the time r
ange to query. Can be any kind of datetime string. For example: 'April 23, 2014'
or 'yesterday'.">[?]</span></label>
<div class="rel-box">
<input id="start" type="text" />
<button type="button" class="rel-btn" data-target="start-select">	
660;</button>
<div id="start-select" class="rel-dropdown" data-input="start">
<div data-value="-1 hour">-1 hour</div>
<div data-value="-3 hours">-3 hours</div>
<div data-value="-12 hours">-12 hours</div>
<div data-value="-1 day">-1 day</div>
<div data-value="-3 days">-3 days</div>
<div data-value="-1 week">-1 week</div>
<div data-value="-1 fortnight">-1 fortnight</div>
<div data-value="-30 days">-30 days</div>
bash-5.2#I need to add a dropdown next to "Time Column" for selecting units (s/ms/us/ns).
bash-5.2# grep -n "Time Column" -n scubaduck/static/js -r
bash-5.2#I'll look for 'time unit' or 'time_units' in the code.
bash-5.2# grep -n "time units" -r scubaduckbash-5.2# grep -n "ms" -r scubaduck | head
scubaduck/static/js/timeseries_chart.js:121: {unit: 'second', step: 1, ms: 10
00},
scubaduck/static/js/timeseries_chart.js:122: {unit: 'second', step: 2, ms: 20
00},
scubaduck/static/js/timeseries_chart.js:123: {unit: 'second', step: 5, ms: 50
00},
scubaduck/static/js/timeseries_chart.js:124: {unit: 'second', step: 10, ms: 1
0000},
scubaduck/static/js/timeseries_chart.js:125: {unit: 'second', step: 15, ms: 1
5000},
scubaduck/static/js/timeseries_chart.js:126: {unit: 'second', step: 30, ms: 3
0000},
scubaduck/static/js/timeseries_chart.js:127: {unit: 'minute', step: 1, ms: 60
000},
scubaduck/static/js/timeseries_chart.js:128: {unit: 'minute', step: 2, ms: 12
0000},
scubaduck/static/js/timeseries_chart.js:129: {unit: 'minute', step: 5, ms: 30
0000},
scubaduck/static/js/timeseries_chart.js:130: {unit: 'minute', step: 10, ms: 6
00000},
bash-5.2#I just need to update HTML/JS for a dropdown and adjust the backend for granularity.
bash-5.2# grep -n "granularity" -r scubaduck/static | head
scubaduck/static/index.html:247: <div id="granularity_field" class="field
" style="display:none;">
scubaduck/static/index.html:249: <select id="granularity">
scubaduck/static/index.html:484: document.getElementById('granularity_field').s
tyle.display = showTS ? 'flex' : 'none';
scubaduck/static/index.html:992: payload.granularity = document.getElementByI
d('granularity').value;
scubaduck/static/index.html:1018: if (params.granularity) sp.set('granularity
', params.granularity);
scubaduck/static/index.html:1041: if (params.granularity) document.getElementBy
Id('granularity').value = params.granularity;
scubaduck/static/index.html:1101: if (sp.has('granularity')) params.granularity
= sp.get('granularity');
bash-5.2#I'll open index.html to inspect the fields around the time column.
bash-5.2# sed -n '232,270p' scubaduck/static/index.html
</div>
</div>
<div class="field">
<label>Order By<span class="help" title="Choose a column to sort resul
ts by.">[?]</span></label>
<select id="order_by"></select>
<button id="order_dir" type="button">ASC \u25B2</button>
</div>
<div class="field">
<label>Limit<span class="help" title="Choose the maximum number of res
ults to show in the chart after any aggregations have been applied. For example,
a limit of 10 will show no more than 10 rows for a table, etc.">[?]</span></lab
el>
<input id="limit" type="number" value="100" />
</div>
<div id="x_axis_field" class="field" style="display:none;">
<label>X-axis</label>
<select id="x_axis"></select>
</div>
<div id="granularity_field" class="field" style="display:none;">
<label>Granularity</label>
<select id="granularity">
<option>Auto</option>
<option>Fine</option>
<option>1 second</option>
<option>5 seconds</option>
<option>10 seconds</option>
<option>30 seconds</option>
<option>1 minute</option>
<option>4 minutes</option>
<option>5 minutes</option>
<option>10 minutes</option>
<option>15 minutes</option>
<option>30 minutes</option>
<option>1 hour</option>
<option>3 hours</option>
<option>6 hours</option>
<option>1 day</option>
<option>1 week</option>
<option>30 days</option>
</select>
</div>
<div id="fill_field" class="field" style="display:none;">
bash-5.2#It looks like I need to go a bit earlier to gather more context.
bash-5.2# sed -n '188,236p' scubaduck/static/index.html
<button class="tab active" data-tab="settings">View Settings</button>
<button class="tab" data-tab="columns">Columns</button>
<button id="dive" onclick="dive()">Dive</button>
</div>
<div id="settings" class="tab-content active">
<div class="field">
<label>Time Column</label>
<select id="time_column"></select>
</div>
<div class="field">
<label>Start<span class="help" title="Sets the start/end of the time r
ange to query. Can be any kind of datetime string. For example: 'April 23, 2014'
or 'yesterday'.">[?]</span></label>
<div class="rel-box">
<input id="start" type="text" />
<button type="button" class="rel-btn" data-target="start-select">	
660;</button>
<div id="start-select" class="rel-dropdown" data-input="start">
<div data-value="-1 hour">-1 hour</div>
<div data-value="-3 hours">-3 hours</div>
<div data-value="-12 hours">-12 hours</div>
<div data-value="-1 day">-1 day</div>
<div data-value="-3 days">-3 days</div>
<div data-value="-1 week">-1 week</div>
<div data-value="-1 fortnight">-1 fortnight</div>
<div data-value="-30 days">-30 days</div>
<div data-value="-90 days">-90 days</div>
</div>
</div>
</div>
<div class="field">
<label>End<span class="help" title="Sets the start/end of the time ran
ge to query. Can be any kind of datetime string. For example: 'April 23, 2014' o
r 'yesterday'.">[?]</span></label>
<div class="rel-box">
<input id="end" type="text" />
<button type="button" class="rel-btn" data-target="end-select">φ
0;</button>
<div id="end-select" class="rel-dropdown" data-input="end">
<div data-value="now">now</div>
<div data-value="-1 hour">-1 hour</div>
<div data-value="-3 hours">-3 hours</div>
<div data-value="-12 hours">-12 hours</div>
<div data-value="-1 day">-1 day</div>
<div data-value="-3 days">-3 days</div>
<div data-value="-1 week">-1 week</div>
<div data-value="-1 fortnight">-1 fortnight</div>
<div data-value="-30 days">-30 days</div>
<div data-value="-90 days">-90 days</div>
</div>
</div>
</div>
<div class="field">
<label>Order By<span class="help" title="Choose a column to sort resul
ts by.">[?]</span></label>
<select id="order_by"></select>
bash-5.2#The user wants a dropdown for selecting time units like s/ms/us/ns, which needs backend handling. I'll check the QueryParams dataclass for parsing.
bash-5.2# grep -n "QueryParams" -n scubaduck/server.py
29:class QueryParams:
215:def build_query(params: QueryParams, column_types: Dict[str, str] | None = N
one) -> str:
437: params = QueryParams(bash-5.2# sed -n '20,80p' scubaduck/server.py
@dataclass
class Filter:
column: str
op: str
value: str | int | float | list[str] | None
@dataclass
class QueryParams:
start: str | None = None
end: str | None = None
order_by: str | None = None
order_dir: str = "ASC"
limit: int | None = None
columns: list[str] = field(default_factory=lambda: [])
filters: list[Filter] = field(default_factory=lambda: [])
derived_columns: dict[str, str] = field(default_factory=lambda: {})
graph_type: str = "samples"
group_by: list[str] = field(default_factory=lambda: [])
aggregate: str | None = None
show_hits: bool = False
x_axis: str | None = None
granularity: str = "Auto"
fill: str = "0"
table: str = "events"
time_column: str = "timestamp"
def _normalize_sqlite_type(sql: str) -> str:
"""Map arbitrary SQLite column types to DuckDB-compatible types."""
t = sql.strip().upper()
if "(" in t:
t = t.split("(", 1)[0]
if "INT" in t:
# SQLite only has a single INTEGER type which is always 64-bit.
# Use DuckDB's BIGINT to avoid overflow when values exceed INT32.
return "BIGINT"
if any(key in t for key in ("CHAR", "CLOB", "TEXT")):
return "VARCHAR"
if "BLOB" in t:
return "BLOB"
if any(key in t for key in ("DOUBLE", "REAL", "FLOA", "NUMERIC", "DECIMAL"))
:
return "DOUBLE"
if "BOOL" in t:
return "BOOLEAN"
if "DATE" in t or "TIME" in t:
return "TIMESTAMP" if "TIME" in t else "DATE"
return "VARCHAR"
def _load_database(path: Path) -> duckdb.DuckDBPyConnection:
if not path.exists():
raise FileNotFoundError(path)
ext = path.suffix.lower()
if ext == ".csv":
con = duckdb.connect()
con.execute(
f"CREATE TABLE events AS SELECT * FROM read_csv_auto('{path.as_posix
()}')"
)
bash-5.2#I'm checking for the QueryParams parsing section now.
bash-5.2# sed -n '420,470p' scubaduck/server.py
rows = con.execute(
f"SELECT DISTINCT {column} FROM \"{table}\" WHERE CAST({column} AS V
ARCHAR) ILIKE '%' || ? || '%' LIMIT 20",
[substr],
).fetchall()
values = [r[0] for r in rows]
_cache_set(key, values)
return jsonify(values)
@app.route("/api/query", methods=["POST"])
def query() -> Any: # pyright: ignore[reportUnusedFunction]
payload = request.get_json(force=True)
try:
start = parse_time(payload.get("start"))
end = parse_time(payload.get("end"))
except Exception as exc:
return jsonify({"error": str(exc)}), 400
params = QueryParams(
start=start,
end=end,
order_by=payload.get("order_by"),
order_dir=payload.get("order_dir", "ASC"),
limit=payload.get("limit"),
columns=payload.get("columns", []),
derived_columns=payload.get("derived_columns", {}),
graph_type=payload.get("graph_type", "samples"),
group_by=payload.get("group_by", []),
aggregate=payload.get("aggregate"),
show_hits=payload.get("show_hits", False),
x_axis=payload.get("x_axis"),
granularity=payload.get("granularity", "Auto"),
fill=payload.get("fill", "0"),
table=payload.get("table", default_table),
time_column=payload.get("time_column", "timestamp"),
)
for f in payload.get("filters", []):
params.filters.append(Filter(f["column"], f["op"], f.get("value")))
if params.table not in tables:
return jsonify({"error": "Invalid table"}), 400
column_types = get_columns(params.table)
if params.time_column not in column_types:
return jsonify({"error": "Invalid time_column"}), 400
if params.graph_type not in {"table", "timeseries"} and (
params.group_by or params.aggregate or params.show_hits
):
return (
jsonify(bash-5.2# sed -n '470,530p' scubaduck/server.py
jsonify(
{
"error": "group_by, aggregate and show_hits are only val
id for table or timeseries view"
}
),
400,
)
valid_cols = set(column_types.keys())
valid_cols.update(params.derived_columns.keys())
if params.graph_type == "timeseries":
if params.x_axis is None:
params.x_axis = params.time_column
if params.x_axis not in valid_cols:
return jsonify({"error": "Invalid x_axis"}), 400
ctype = column_types.get(params.x_axis, "").upper()
is_time = any(t in ctype for t in ["TIMESTAMP", "DATE", "TIME"])
is_numeric = any(
t in ctype
for t in [
"INT",
"DECIMAL",
"REAL",
"DOUBLE",
"FLOAT",
"NUMERIC",
"HUGEINT",
]
)
if not (is_time or is_numeric):
return jsonify({"error": "x_axis must be a time column"}), 400
for col in params.columns:
if col not in valid_cols:
return jsonify({"error": f"Unknown column: {col}"}), 400
for col in params.group_by:
if col not in valid_cols:
return jsonify({"error": f"Unknown column: {col}"}), 400
if params.order_by and params.order_by not in valid_cols:
return jsonify({"error": f"Unknown column: {params.order_by}"}), 400
if params.group_by or params.graph_type == "timeseries":
agg = (params.aggregate or "avg").lower()
if agg.startswith("p") or agg == "sum":
need_numeric = True
allow_time = False
elif agg == "avg" or agg in {"min", "max"}:
need_numeric = False
allow_time = True
else:
need_numeric = False
allow_time = False
if need_numeric or allow_time:
for c in params.columns:
if c in params.group_by or c == params.x_axis:
continue
if c not in column_types:
continue
ctype = column_types.get(c, "").upper()
is_numeric = any(
t in ctype
for t in [bash-5.2# sed -n '530,590p' scubaduck/server.py
for t in [
"INT",
"DECIMAL",
"REAL",
"DOUBLE",
"FLOAT",
"NUMERIC",
"HUGEINT",
]
)
is_time = "TIMESTAMP" in ctype or "DATE" in ctype or "TIME"
in ctype
if need_numeric and not is_numeric:
return (
jsonify(
{
"error": f"Aggregate {agg} cannot be applied
to column {c}",
}
),
400,
)
if allow_time and not (is_numeric or is_time):
return (
jsonify(
{
"error": f"Aggregate {agg} cannot be applied
to column {c}",
}
),
400,
)
if params.start is None or params.end is None:
axis = params.x_axis or params.time_column
row = cast(
tuple[datetime | None, datetime | None],
con.execute(
f'SELECT min({axis}), max({axis}) FROM "{params.table}"'
).fetchall()[0],
)
mn, mx = row
if isinstance(mn, (int, float)):
mn = datetime.fromtimestamp(int(mn), tz=timezone.utc)
if isinstance(mx, (int, float)):
mx = datetime.fromtimestamp(int(mx), tz=timezone.utc)
if params.start is None and mn is not None:
params.start = (
mn.strftime("%Y-%m-%d %H:%M:%S") if not isinstance(mn, str)
else mn
)
if params.end is None and mx is not None:
params.end = (
mx.strftime("%Y-%m-%d %H:%M:%S") if not isinstance(mx, str)
else mx
)
bucket_size: int | None = None
if params.graph_type == "timeseries":
bucket_size = _granularity_seconds(
params.granularity,
params.start if isinstance(params.start, str) else None,
params.end if isinstance(params.end, str) else None,
)
if (
params.limit is not None
and params.start is not Nonebash-5.2# sed -n '590,650p' scubaduck/server.py
and params.start is not None
and params.end is not None
):
try:
start_dt = dtparser.parse(params.start)
end_dt = dtparser.parse(params.end)
buckets = math.ceil(
(end_dt - start_dt).total_seconds() / bucket_size
)
if buckets > 1:
params.limit *= buckets
except Exception:
pass
sql = build_query(params, column_types)
try:
rows = con.execute(sql).fetchall()
except Exception as exc:
tb = traceback.format_exc()
print(f"Query failed:\n{sql}\n{tb}")
return (
jsonify({"sql": sql, "error": str(exc), "traceback": tb}),
400,
)
result: Dict[str, Any] = {"sql": sql, "rows": rows}
if params.start is not None:
result["start"] = str(params.start)
if params.end is not None:
result["end"] = str(params.end)
if bucket_size is not None:
result["bucket_size"] = bucket_size
return jsonify(result)
return app
app = create_app()
if __name__ == "__main__":
app.run(debug=True)
bash-5.2#I'll open tests/test_server.py to examine its features more closely.
bash-5.2# ls tests
conftest.py test_server.py test_web.pybash-5.2# sed -n '1,200p' tests/test_server.py
from __future__ import annotations
import json
from pathlib import Path
from typing import cast
import duckdb
from scubaduck import server
import pytest
def test_basic_query() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-02 00:00:00",
"order_by": "timestamp",
"order_dir": "ASC",
"limit": 10,
"columns": ["timestamp", "event", "value", "user"],
"filters": [],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert data
rows = data["rows"]
# We expect first three rows (until 2024-01-02 00:00:00)
assert len(rows) == 3
assert rows[0][1] == "login"
assert rows[1][1] == "logout"
def test_js_served() -> None:
app = server.app
client = app.test_client()
rv = client.get("/js/chip_input.js")
assert rv.status_code == 200
assert b"initChipInput" in rv.data
def test_filter_multi_token() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-02 03:00:00",
"order_by": "timestamp",
"limit": 10,
"columns": ["timestamp", "event", "value", "user"],
"filters": [{"column": "user", "op": "=", "value": ["alice", "charlie"]}
],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert data
rows = data["rows"]
# Should only return rows for alice and charlie
assert len(rows) == 3
assert rows[0][3] == "alice"
assert rows[-1][3] == "charlie"
def test_empty_filter_is_noop() -> None:
app = server.app
client = app.test_client()
base_payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"limit": 100,
"columns": ["timestamp", "event", "value", "user"],
}
no_filter = {**base_payload, "filters": []}
empty_filter = {
**base_payload,
"filters": [{"column": "user", "op": "=", "value": None}],
}
rv1 = client.post(
"/api/query", data=json.dumps(no_filter), content_type="application/json
"
)
rv2 = client.post(
"/api/query", data=json.dumps(empty_filter), content_type="application/j
son"
)
rows1 = rv1.get_json()["rows"]
rows2 = rv2.get_json()["rows"]
assert rows1 == rows2
def test_select_columns() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"order_by": "timestamp",
"limit": 10,
"columns": ["timestamp", "user"],
"filters": [],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert data
rows = data["rows"]
assert len(rows[0]) == 2
assert rows[0][1] == "alice"
def test_string_filter_ops() -> None:
app = server.app
client = app.test_client()
base = {
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"order_by": "timestamp",
"limit": 100,
"columns": ["timestamp", "event", "value", "user"],
}
contains = {
**base,
"filters": [{"column": "user", "op": "contains", "value": "ali"}],
}
rv = client.post(
"/api/query", data=json.dumps(contains), content_type="application/json"
)
rows = rv.get_json()["rows"]
assert all("ali" in r[3] for r in rows)
regex = {
**base,
"filters": [{"column": "user", "op": "~", "value": "^a.*"}],
}
rv = client.post(
"/api/query", data=json.dumps(regex), content_type="application/json"
)
rows = rv.get_json()["rows"]
assert all(r[3].startswith("a") for r in rows)
assert len(rows) == 2
not_empty = {**base, "filters": [{"column": "user", "op": "!empty"}]}
rv = client.post(
"/api/query", data=json.dumps(not_empty), content_type="application/json
"
)
assert len(rv.get_json()["rows"]) == 4
def _make_payload() -> dict[str, object]:
return {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-02 00:00:00",
"order_by": "timestamp",
"order_dir": "ASC",
"limit": 10,
"columns": ["timestamp", "event", "value", "user"],
"filters": [],
}
def test_database_types(tmp_path: Path) -> None:
csv_file = tmp_path / "events.csv"
csv_file.write_text(Path("scubaduck/sample.csv").read_text())
sqlite_file = tmp_path / "events.sqlite"
import sqlite3
conn = sqlite3.connect(sqlite_file)
conn.execute(
"CREATE TABLE events (timestamp TEXT, event TEXT, value INTEGER, user TE
XT)"
)
with open(csv_file) as f:
next(f)
for line in f:
ts, ev, val, user = line.strip().split(",")
conn.execute(
"INSERT INTO events VALUES (?, ?, ?, ?)", (ts, ev, int(val), use
r)
)
conn.commit()
conn.close() # pyright: ignore[reportUnknownMemberType, reportAttributeAcce
ssIssue]
duckdb_file = tmp_path / "events.duckdb"
con = duckdb.connect(duckdb_file)
con.execute(
f"CREATE TABLE events AS SELECT * FROM read_csv_auto('{csv_file.as_posix
()}')"
)
con.close() # pyright: ignore[reportUnknownMemberType, reportAttributeAcces
sIssue]
for db in (csv_file, sqlite_file, duckdb_file):
app = server.create_app(db)
client = app.test_client()
bash-5.2#I’m continuing to scroll through for any relevant updates.
on"
)
rows = rv.get_json()["rows"]
assert len(rows) == 3
def test_sqlite_longvarchar(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) ->
None:
sqlite_file = tmp_path / "events.sqlite"
import sqlite3
conn = sqlite3.connect(sqlite_file)
conn.execute(
"CREATE TABLE events (timestamp TEXT, url LONGVARCHAR, title VARCHAR(10)
)"
)
conn.execute(
"INSERT INTO events VALUES ('2024-01-01 00:00:00', 'https://a.com', 'Hom
e')"
)
conn.commit()
conn.close() # pyright: ignore[reportUnknownMemberType, reportAttributeAcce
ssIssue]
from typing import Any
real_connect = duckdb.connect
def failing_connect(*args: Any, **kwargs: Any) -> Any:
real = real_connect(*args, **kwargs)
class Wrapper:
def __init__(self, con: duckdb.DuckDBPyConnection) -> None:
self.con = con
self._failed = False
def execute(self, sql: str, *a: Any, **kw: Any):
if not self._failed and sql == "LOAD sqlite":
self._failed = True
raise RuntimeError("fail")
return self.con.execute(sql, *a, **kw)
def __getattr__(self, name: str) -> object:
return getattr(self.con, name)
return Wrapper(real)
monkeypatch.setattr(server.duckdb, "connect", failing_connect)
app = server.create_app(sqlite_file)
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-01 01:00:00",
"order_by": "timestamp",
"columns": ["timestamp", "url", "title"],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert data["rows"][0][1] == "https://a.com"
def test_sqlite_bigint(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
sqlite_file = tmp_path / "big.sqlite"
import sqlite3
conn = sqlite3.connect(sqlite_file)
conn.execute("CREATE TABLE events (timestamp TEXT, value INTEGER)")
big_value = 13385262862605259
conn.execute(
"INSERT INTO events VALUES ('2024-01-01 00:00:00', ?)",
(big_value,),
)
conn.commit()
conn.close() # pyright: ignore[reportUnknownMemberType, reportAttributeAcce
ssIssue]
from typing import Any
real_connect = duckdb.connect
def failing_connect(*args: Any, **kwargs: Any) -> Any:
real = real_connect(*args, **kwargs)
class Wrapper:
def __init__(self, con: duckdb.DuckDBPyConnection) -> None:
self.con = con
self._failed = False
def execute(self, sql: str, *a: Any, **kw: Any):
if not self._failed and sql == "LOAD sqlite":
self._failed = True
raise RuntimeError("fail")
return self.con.execute(sql, *a, **kw)
def __getattr__(self, name: str) -> object:
return getattr(self.con, name)
return Wrapper(real)
monkeypatch.setattr(server.duckdb, "connect", failing_connect)
app = server.create_app(sqlite_file)
client = app.test_client()
payload = {
"table": "events",
"order_by": "timestamp",
"columns": ["timestamp", "value"],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert data["rows"][0][1] == big_value
def test_integer_time_column(tmp_path: Path) -> None:
csv_file = tmp_path / "events.csv"
csv_file.write_text("created,event\n1704067200,login\n1704070800,logout\n")
app = server.create_app(csv_file)
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-01 01:00:00",
"order_by": "created",
"columns": ["created", "event"],
"time_column": "created",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert len(data["rows"]) == 2
def test_envvar_db(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
csv_file = tmp_path / "custom.csv"
csv_file.write_text("timestamp,event,value,user\n2024-01-01 00:00:00,login,5
,bob\n")
monkeypatch.setenv("SCUBADUCK_DB", str(csv_file))
app = server.create_app()
client = app.test_client()
payload = _make_payload()
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
rows = rv.get_json()["rows"]
assert len(rows) == 1
def test_envvar_db_missing(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> N
one:
missing = tmp_path / "missing.sqlite"
monkeypatch.setenv("SCUBADUCK_DB", str(missing))
with pytest.raises(FileNotFoundError):
server.create_app()
def test_group_by_table() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "table",
"order_by": "user",
"limit": 10,
"columns": ["value"],
"group_by": ["user"],
"aggregate": "Sum",
"show_hits": True,
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
rows = rv.get_json()["rows"]
assert rows[0][0] == "alice"
assert rows[0][1] == 2
assert rows[0][2] == 40
def test_relative_time_query(monkeypatch: pytest.MonkeyPatch) -> None:
app = server.app
client = app.test_client()
from datetime import datetime
fixed_now = datetime(2024, 1, 2, 4, 0, 0)
class FixedDateTime(datetime):
@classmethod
def now(cls, tz=None): # type: ignore[override]
return fixed_now if tz is None else fixed_now.astimezone(tz)
monkeypatch.setattr(server, "datetime", FixedDateTime)
payload = {data = rv.get_json()
assert rv.status_code == 200
assert len(data["rows"]) == 1
assert data["rows"][0][3] == "charlie"
def test_invalid_time_error() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "nonsense",
"end": "now",
"order_by": "timestamp",
"limit": 10,
"columns": ["timestamp"],
"filters": [],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 400
assert "error" in data
def test_query_error_returns_sql_and_traceback() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"columns": ["event"],
"group_by": ["user"],
"aggregate": "avg",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 400
assert "error" in data
def test_table_unknown_column_error() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "table",
"order_by": "timestamp",
"limit": 100,
"columns": ["user", "Hits", "value"],
"group_by": ["user"],
"aggregate": "Count",
"show_hits": True,
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 400
assert "Unknown column" in data["error"]
def test_samples_view_rejects_group_by() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"graph_type": "samples",
"group_by": ["user"],
"columns": ["timestamp"],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 400
assert "only valid" in data["error"]
def test_table_avg_with_timestamp() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "table",
"order_by": "user",
"limit": 100,
"columns": ["user", "timestamp", "value"],
"group_by": ["user"],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert "error" not in data
rows = data["rows"]
assert rows[0][0] == "alice"
from dateutil import parser
ts = parser.parse(rows[0][1]).replace(tzinfo=None)
assert ts == parser.parse("2024-01-01 12:00:00")
def test_timeseries_basic() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "timeseries",
"limit": 100,
"group_by": ["user"],
"aggregate": "Count",
"columns": ["value"],
...
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert len(data["rows"]) == 2
assert all(r[1] == "alice" for r in data["rows"])
def test_timeseries_auto_and_fine_buckets() -> None:
app = server.app
client = app.test_client()
def run(gran: str) -> None:
payload = {
"start": "2024-01-01 00:00:00",
"end": "2024-01-02 03:00:00",
"graph_type": "timeseries",
"columns": ["value"],
"x_axis": "timestamp",
"granularity": gran,
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/js
on"
)
data = rv.get_json()
assert rv.status_code == 200
from dateutil import parser
start = parser.parse(cast(str, payload["start"])).replace(tzinfo=None)
buckets = [
parser.parse(cast(str, r[0])).replace(tzinfo=None) for r in data["ro
ws"]
]
assert buckets[0] == start
if len(buckets) > 1:
step = (buckets[1] - buckets[0]).total_seconds()
assert step % data["bucket_size"] == 0
assert any(r[1] != 0 for r in data["rows"])
run("Auto")
run("Fine")
def test_timeseries_string_column_error() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "timeseries",
"limit": 7,
"columns": ["timestamp", "event", "value", "user"],
"x_axis": "timestamp",
"granularity": "1 hour",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 400
assert "Aggregate" in data["error"]
def test_derived_column_basic() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"limit": 10,
"columns": ["timestamp"],
"derived_columns": {"val2": "value * 2"},
"filters": [],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert data["rows"][0][1] == 20
def test_timeseries_derived_column() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "timeseries",
"granularity": "1 hour",
"limit": 7,
"columns": ["value"],
"derived_columns": {"derived_1": "value * 2"},
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
rows = data["rows"]
assert all(r[2] == r[1] * 2 for r in rows)
def test_default_start_end_returned() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"order_by": "timestamp",
"limit": 5,
"columns": ["timestamp"],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert data["start"] == "2024-01-01 00:00:00"
assert data["end"] == "2024-01-02 03:00:00"
bash-5.2#The test file is lengthy; I'll check tests/test_web.py for relevant tests.
def run_query(
page: Any,
url: str,
*,
start: str | None = None,
end: str | None = None,
order_by: str | None = None,
order_dir: str | None = "ASC",
limit: int | None = None,
group_by: list[str] | None = None,
aggregate: str | None = None,
) -> dict[str, Any]:
page.goto(url)
page.wait_for_selector("#order_by option", state="attached")
page.wait_for_selector("#order_dir", state="attached")
page.wait_for_function("window.lastResults !== undefined")
if start is not None:
page.fill("#start", start)
if end is not None:
page.fill("#end", end)
if order_by is not None:
select_value(page, "#order_by", order_by)
if order_dir is not None and order_dir == "DESC":
page.click("#order_dir")
if limit is not None:
page.fill("#limit", str(limit))
if group_by is not None:
select_value(page, "#graph_type", "table")
page.evaluate(
"g => { groupBy.chips = g; groupBy.renderChips(); }",
group_by,
)
if aggregate is not None:
select_value(page, "#graph_type", "table")
select_value(page, "#aggregate", aggregate)
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
return page.evaluate("window.lastResults")
def test_range_filters(page: Any, server_url: str) -> None:
data = run_query(
page,
server_url,
start="2024-01-02 00:00:00",
end="2024-01-02 04:00:00",
order_by="timestamp",
limit=100,
)
assert len(data["rows"]) == 2
from dateutil import parser
timestamps = [parser.parse(row[0]).replace(tzinfo=None) for row in data["row
s"]]
assert timestamps == [
parser.parse("2024-01-02 00:00:00"),
parser.parse("2024-01-02 03:00:00"),
]
def test_order_by(page: Any, server_url: str) -> None:
data = run_query(
page,
server_url,
start="2024-01-01 00:00:00",
end="2024-01-03 00:00:00",
order_by="value",
order_dir="DESC",
limit=100,
)
values = [row[2] for row in data["rows"]]
assert values == sorted(values, reverse=True)
def test_limit(page: Any, server_url: str) -> None:
data = run_query(
page,
server_url,
start="2024-01-01 00:00:00",
end="2024-01-03 00:00:00",
order_by="timestamp",
limit=2,
)
assert len(data["rows"]) == 2
def test_time_column_dropdown(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#time_column option", state="attached")
options = page.locator("#time_column option").all_inner_texts()
assert "timestamp" in options
assert "value" in options
assert page.input_value("#time_column") == "timestamp"
def test_simple_filter(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
page.click("text=Add Filter")
filter_el = page.query_selector("#filters .filter:last-child")
assert filter_el
page.evaluate(
"arg => setSelectValue(arg.el.querySelector('.f-col'), arg.val)",
{"el": filter_el, "val": "user"},
)
val_input = filter_el.query_selector(".f-val")
val_input.click()
page.keyboard.type("alice")
page.keyboard.press("Enter")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
data = page.evaluate("window.lastResults")
assert len(data["rows"]) == 2
assert all(row[3] == "alice" for row in data["rows"])
def test_default_filter_and_layout(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
count = page.evaluate("document.querySelectorAll('#filters .filter').length"
)
assert count == 1
last_is_button = page.evaluate(
"document.querySelector('#filters').lastElementChild.id === 'add_filter'
"
)
assert last_is_button
position = page.evaluate(
"getComputedStyle(document.querySelector('#filters .filter button.remove
')).position"
)
assert position == "static"
def test_filter_remove_alignment(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
diff = page.evaluate(
"() => { const r=document.querySelector('#filters .filter-row').getBound
ingClientRect(); const x=document.querySelector('#filters .filter-row button.rem
ove').getBoundingClientRect(); return Math.abs(r.right - x.right); }"
)
assert diff <= 1
def test_header_and_tabs(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
header = page.text_content("#header")
assert "sample.csv" in header
assert "events" in header
assert page.is_visible("#settings")
assert page.is_hidden("#columns")
page.click("text=Columns")
assert page.is_visible("#columns")
cols = [c.strip() for c in page.locator("#column_groups li").all_inner_texts
()]
assert "timestamp" in cols
assert "event" in cols
page.click("text=View Settings")
assert page.is_visible("#settings")
btn_color = page.evaluate(
"getComputedStyle(document.querySelector('#dive')).backgroundColor"
)
assert "rgb(0, 128, 0)" == btn_color
sidebar_overflow = page.evaluate(
"getComputedStyle(document.querySelector('#sidebar')).overflowY"
)
view_overflow = page.evaluate(
"getComputedStyle(document.querySelector('#view')).overflowY"
)
assert sidebar_overflow == "auto"
assert view_overflow == "auto"
def test_graph_type_table_fields(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#graph_type", state="attached")
select_value(page, "#graph_type", "table")
assert page.is_visible("#group_by_field")
assert page.is_visible("#aggregate_field")
assert page.is_visible("#show_hits_field")
page.click("text=Columns")
assert not page.is_visible("text=Strings:")
def test_graph_type_timeseries_fields(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#graph_type", state="attached")
select_value(page, "#graph_type", "timeseries")
bash-5.2#I need to scroll further to check for tests related to time columns.
bash-5.2# grep -n "time_column" -n tests/test_web.py
99:def test_time_column_dropdown(page: Any, server_url: str) -> None:
101: page.wait_for_selector("#time_column option", state="attached")
102: options = page.locator("#time_column option").all_inner_texts()
105: assert page.input_value("#time_column") == "timestamp"
bash-5.2#I'm opening timeseries_chart.js to check how times are being handled for the dropdown feature.
let currentChart = null;
function showTimeSeries(data) {
function parseTs(s) {
if (s.match(/GMT/) || s.endsWith('Z') || /\+\d{2}:?\d{2}$/.test(s)) {
return new Date(s).getTime();
}
return new Date(s + 'Z').getTime();
}
const view = document.getElementById('view');
if (data.rows.length === 0) {
view.innerHTML = '<p id="empty-message">Empty data provided to table</p>';
return;
}
const height = 600;
view.innerHTML =
'<div id="ts-container"><div id="legend"></div><div id="chart-wrapper"><svg
id="chart" height="' +
height +
'"></svg></div></div>';
const svg = document.getElementById('chart');
const legend = document.getElementById('legend');
const crosshairLine = document.createElementNS('http://www.w3.org/2000/svg', '
line');
crosshairLine.id = 'crosshair_line';
crosshairLine.setAttribute('stroke', '#555');
crosshairLine.style.display = 'none';
const crosshairDots = document.createElementNS('http://www.w3.org/2000/svg', '
g');
crosshairDots.id = 'crosshair_dots';
crosshairDots.style.display = 'none';
const groups = groupBy.chips || [];
const hasHits = document.getElementById('show_hits').checked ? 1 : 0;
const fill = document.getElementById('fill').value;
const bucketMs = (data.bucket_size || 3600) * 1000;
const start = data.start ? parseTs(data.start) : null;
const end = data.end ? parseTs(data.end) : null;
const startIdx = 1 + groups.length + hasHits;
const valueCols = selectedColumns.slice(groups.length + hasHits);
const series = {};
data.rows.forEach(r => {
bash-5.2#I'll check out the isTimeColumn function in the index.html for more insight.
bash-5.2# grep -n "isTimeColumn" -n scubaduck/static/index.html
497: if (isTimeColumn(cb.value) || isStringColumn(cb.value)) {
746:function isTimeColumn(name) {
1172: if (isTimeColumn(col)) {bash-5.2# sed -n '726,770p' scubaduck/static/index.html
if (dc.include && !selectedColumns.includes(dc.name)) selectedColumns.push
(dc.name);
});
} else {
selectedColumns = base.slice();
derivedColumns.forEach(dc => {
if (dc.include) selectedColumns.push(dc.name);
});
}
}
function isStringColumn(name) {
const t = (columnTypes[name] || '').toUpperCase();
return t.includes('CHAR') || t.includes('STRING') || t.includes('VARCHAR');
}
function isIntegerColumn(name) {
const t = (columnTypes[name] || '').toUpperCase();
return t.includes('INT');
}
function isTimeColumn(name) {
const t = (columnTypes[name] || '').toUpperCase();
if (t.includes('TIMESTAMP') || t.includes('DATE') || t.includes('TIME')) retur
n true;
const sel = document.getElementById('time_column').value;
const xsel = document.getElementById('x_axis').value;
if (name === sel || name === xsel) return true;
return false;
}
function formatNumber(val) {
if (typeof val !== 'number') val = Number(val);
if (Number.isNaN(val)) return '';
if (val === 0) return '0';
const abs = Math.abs(val);
if (abs > 999.999) {
const units = [
{n: 1e12, s: 'T'},
{n: 1e9, s: 'B'},
{n: 1e6, s: 'M'},
{n: 1e3, s: 'K'},
];
for (const u of units) {
if (abs >= u.n) {
return (val / u.n).toFixed(2) + ' ' + u.s;
}
bash-5.2#Looking into time unit selection for integer columns and updating backend parameters to match.
bash-5.2# grep -n "granularity" -n scubaduck/static/index.html
247: <div id="granularity_field" class="field" style="display:none;">
249: <select id="granularity">
484: document.getElementById('granularity_field').style.display = showTS ? 'fle
x' : 'none';
992: payload.granularity = document.getElementById('granularity').value;
1018: if (params.granularity) sp.set('granularity', params.granularity);
1041: if (params.granularity) document.getElementById('granularity').value = pa
rams.granularity;
1101: if (sp.has('granularity')) params.granularity = sp.get('granularity');
bash-5.2#I'll look for references to 'timeColumns' or 'baseTimeColumnOptions' to better understand.
bash-5.2# grep -n "baseTimeColumns" -n scubaduck/static/index.html
349:const baseTimeColumns = [];
531: baseTimeColumns.length = 0;
557: baseTimeColumns.push(c.name);
902: timeColumns.splice(0, timeColumns.length, ...baseTimeColumns);bash-5.2# sed -n '332,362p' scubaduck/static/index.html
</div>
<div id="sidebar-resizer"></div>
<div id="view">
<table id="results"></table>
</div>
</div>
<script src="/js/chip_input.js"></script>
<script src="/js/timeseries_chart.js"></script>
<script>
const allColumns = [];
const baseColumns = [];
const columnTypes = {};
const stringColumns = [];
const baseStringColumns = [];
const integerColumns = [];
const baseIntegerColumns = [];
const timeColumns = [];
const baseTimeColumns = [];
const timeColumnOptions = [];
const baseTimeColumnOptions = [];
const derivedColumns = [];
let selectedColumns = [];
let displayType = 'samples';
let groupBy = {chips: [], addChip: () => {}, renderChips: () => {}};
let defaultTimeColumn = '';
function initDropdown(select) {
const wrapper = document.createElement('div');
wrapper.className = 'dropdown';
if (select.classList.contains('f-col')) {
wrapper.classList.add('f-col');
bash-5.2#I'm checking the lines I spotted earlier, moving further towards the end.
bash-5.2# sed -n '530,566p' scubaduck/static/index.html
baseIntegerColumns.length = 0;
baseTimeColumns.length = 0;
baseTimeColumnOptions.length = 0;
for (const k in columnTypes) delete columnTypes[k];
const groups = {
time: {name: 'Time', cols: [], ul: null},
integer: {name: 'Integers', cols: [], ul: null},
string: {name: 'Strings', cols: [], ul: null},
};
const heur = ['timestamp','created','created_at','event_time','time','date',
'occurred','happened','logged'];
let guess = null;
cols.forEach(c => {
const t = c.type.toUpperCase();
columnTypes[c.name] = c.type;
allColumns.push(c.name);
baseColumns.push(c.name);
let g = 'string';
const isNumeric = t.includes('INT') || t.includes('DECIMAL') || t.includes
('NUMERIC') || t.includes('REAL') || t.includes('DOUBLE') || t.includes('FLOAT')
|| t.includes('HUGEINT');
const isTimeType = t.includes('TIMESTAMP') || t.includes('DATE') || t.incl
udes('TIME');
if (isNumeric || isTimeType) {
timeColumnOptions.push(c.name);
baseTimeColumnOptions.push(c.name);
if (!guess && heur.some(h => c.name.toLowerCase().includes(h))) guess =
c.name;
}
if (isTimeType) {
g = 'time';
timeColumns.push(c.name);
baseTimeColumns.push(c.name);
} else if (isNumeric) {
g = 'integer';
}
if (g === 'string') {
stringColumns.push(c.name);
baseStringColumns.push(c.name);
} else if (g === 'integer') {
integerColumns.push(c.name);
baseIntegerColumns.push(c.name);bash-5.2# sed -n '566,618p' scubaduck/static/index.html
baseIntegerColumns.push(c.name);
}
groups[g].cols.push(c.name);
if (g !== 'string') {
const o = document.createElement('option');
o.value = c.name;
o.textContent = c.name;
orderSelect.appendChild(o);
}
});
timeColumns.forEach(name => {
const o = document.createElement('option');
o.value = name;
o.textContent = name;
xAxisSelect.appendChild(o);
});
timeColumnOptions.forEach(name => {
const o = document.createElement('option');
o.value = name;
o.textContent = name;
timeColumnSelect.appendChild(o);
});
defaultTimeColumn = guess || timeColumnOptions[0] || '';
Object.keys(groups).forEach(key => {
const g = groups[key];
const div = document.createElement('div');
div.className = 'col-group';
const header = document.createElement('div');
header.className = 'col-group-header';
header.appendChild(document.createTextNode(g.name + ': '));
const links = document.createElement('span');
links.className = 'links';
const allBtn = document.createElement('a');
allBtn.href = '#';
allBtn.textContent = 'All';
const noneBtn = document.createElement('a');
noneBtn.href = '#';
noneBtn.textContent = 'None';
links.appendChild(allBtn);
links.appendChild(noneBtn);
header.appendChild(links);
div.appendChild(header);
const ul = document.createElement('ul');
g.ul = ul;
g.cols.forEach(name => {
const li = document.createElement('li');
const label = document.createElement('label');
const cb = document.createElement('input');
cb.type = 'checkbox';
cb.value = name;
cb.checked = true;
cb.addEventListener('change', updateSelectedColumns);
label.appendChild(cb);
bash-5.2#The user wants a dropdown for time units if the column is integer—I'll check the HTML.
bash-5.2# grep -n "time_unit" -r scubaduck
bash-5.2#I need to add a time unit and update the server and tests. Let’s check README.
bash-5.2# cat README.md | head
I really like Scuba (Meta's internal real-time database system). The distributed
, real-time database part of Scuba is quite difficult (and expensive) to replica
te, but I also really like Scuba's UI for doing queries, and I have found myself
wishing that I have access to it even for "small" databases, e.g., I have a sql
ite dataset I want to explore.
Pivotal ideas:
* Time series by default. In the dedicated "time series" view, there are many fe
atures specifically oriented towards working towards tables that represent event
s that occurred over time: the start, end, compare, aggregate and granularity fi
elds all specially privilege the timestamp field. In fact, you can't log events
to Scuba's backing data store without a timestamp, they always come with one. (S
cuba also supports other views that don't presuppose a time series, but the time
series is the most beloved and well used view.) This is in contrast to typical
software which tries to generalize to arbitrary data first, with time series bei
ng added on later.
* It's all about exploration. Scuba is predicated on the idea that you don't kno
w what you're looking for, that you are going to spend time tweaking queries and
changing filters/grouping as part of an investigation to figure out why a syste
m behaves the way it is. So the filters/comparisons/groupings you want to edit a
re always visible on the left sidebar, with the expectation that you're going to
tweak the query to look at something else. Similarly, all the parameters of you
r query get saved into your URL, so your browser history can double up as a quer
y history / you can easily share a query with someone else. This is contrast to
typical software which is often oriented to making pretty dashboards and reports
. (This function is important too, but it's not what I want in exploration mode!
)
* You can fix data problems in the query editor. It's pretty common to have mess
ed up and ended up with a database that doesn't have exactly the columns you nee
d, or some columns that are corrupted in some way. Scuba has pretty robust suppo
rt for defining custom columns with arbitrary SQL functions, grouping over them
as if they were native functions, and doing so with minimal runtime cost (Scuba
aims to turn around your query in milliseconds!) Having to go and run a huge dat
a pipeline to fix your data is a big impediment to exploration; quick and easy c
ustom columns means you can patch over problems when you're investigating and fi
x them for real later.
bash-5.2#I'll add a "time_unit" dropdown next to the "time_column" in the frontend UI.
bash-5.2# grep -n "time_column" -n scubaduck/static/index.html | head
195: <select id="time_column"></select>
518: const timeColumnSelect = document.getElementById('time_column');
749: const sel = document.getElementById('time_column').value;
958: time_column: document.getElementById('time_column').value,
1001: if (params.time_column) sp.set('time_column', params.time_column);
1027: document.getElementById('time_column').value = params.time_column || defa
ultTimeColumn;
1086: if (sp.has('time_column')) params.time_column = sp.get('time_column');
bash-5.2#I'll check around line 518 to see what’s going on there.
bash-5.2# sed -n '512,528p' scubaduck/static/index.html
function loadColumns(table) {
return fetch('/api/columns?table=' + encodeURIComponent(table)).then(r => r.js
on()).then(cols => {
const orderSelect = document.getElementById('order_by');
const xAxisSelect = document.getElementById('x_axis');
const groupsEl = document.getElementById('column_groups');
const timeColumnSelect = document.getElementById('time_column');
orderSelect.innerHTML = '';
xAxisSelect.innerHTML = '';
timeColumnSelect.innerHTML = '';
groupsEl.innerHTML = '';
allColumns.length = 0;
stringColumns.length = 0;
integerColumns.length = 0;
timeColumns.length = 0;
timeColumnOptions.length = 0;
baseColumns.length = 0;bash-5.2# sed -n '528,552p' scubaduck/static/index.html
baseColumns.length = 0;
baseStringColumns.length = 0;
baseIntegerColumns.length = 0;
baseTimeColumns.length = 0;
baseTimeColumnOptions.length = 0;
for (const k in columnTypes) delete columnTypes[k];
const groups = {
time: {name: 'Time', cols: [], ul: null},
integer: {name: 'Integers', cols: [], ul: null},
string: {name: 'Strings', cols: [], ul: null},
};
const heur = ['timestamp','created','created_at','event_time','time','date',
'occurred','happened','logged'];
let guess = null;
cols.forEach(c => {
const t = c.type.toUpperCase();
columnTypes[c.name] = c.type;
allColumns.push(c.name);
baseColumns.push(c.name);
let g = 'string';
const isNumeric = t.includes('INT') || t.includes('DECIMAL') || t.includes
('NUMERIC') || t.includes('REAL') || t.includes('DOUBLE') || t.includes('FLOAT')
|| t.includes('HUGEINT');
const isTimeType = t.includes('TIMESTAMP') || t.includes('DATE') || t.incl
udes('TIME');
if (isNumeric || isTimeType) {
timeColumnOptions.push(c.name);
baseTimeColumnOptions.push(c.name);
if (!guess && heur.some(h => c.name.toLowerCase().includes(h))) guess =
c.name;bash-5.2# sed -n '552,588p' scubaduck/static/index.html
if (!guess && heur.some(h => c.name.toLowerCase().includes(h))) guess =
c.name;
}
if (isTimeType) {
g = 'time';
timeColumns.push(c.name);
baseTimeColumns.push(c.name);
} else if (isNumeric) {
g = 'integer';
}
if (g === 'string') {
stringColumns.push(c.name);
baseStringColumns.push(c.name);
} else if (g === 'integer') {
integerColumns.push(c.name);
baseIntegerColumns.push(c.name);
}
groups[g].cols.push(c.name);
if (g !== 'string') {
const o = document.createElement('option');
o.value = c.name;
o.textContent = c.name;
orderSelect.appendChild(o);
}
});
timeColumns.forEach(name => {
const o = document.createElement('option');
o.value = name;
o.textContent = name;
xAxisSelect.appendChild(o);
});
timeColumnOptions.forEach(name => {
const o = document.createElement('option');
o.value = name;
o.textContent = name;
timeColumnSelect.appendChild(o);
});
defaultTimeColumn = guess || timeColumnOptions[0] || '';bash-5.2# sed -n '588,646p' scubaduck/static/index.html
defaultTimeColumn = guess || timeColumnOptions[0] || '';
Object.keys(groups).forEach(key => {
const g = groups[key];
const div = document.createElement('div');
div.className = 'col-group';
const header = document.createElement('div');
header.className = 'col-group-header';
header.appendChild(document.createTextNode(g.name + ': '));
const links = document.createElement('span');
links.className = 'links';
const allBtn = document.createElement('a');
allBtn.href = '#';
allBtn.textContent = 'All';
const noneBtn = document.createElement('a');
noneBtn.href = '#';
noneBtn.textContent = 'None';
links.appendChild(allBtn);
links.appendChild(noneBtn);
header.appendChild(links);
div.appendChild(header);
const ul = document.createElement('ul');
g.ul = ul;
g.cols.forEach(name => {
const li = document.createElement('li');
const label = document.createElement('label');
const cb = document.createElement('input');
cb.type = 'checkbox';
cb.value = name;
cb.checked = true;
cb.addEventListener('change', updateSelectedColumns);
label.appendChild(cb);
label.appendChild(document.createTextNode(' ' + name));
li.appendChild(label);
ul.appendChild(li);
});
allBtn.addEventListener('click', e => {
e.preventDefault();
ul.querySelectorAll('input').forEach(cb => (cb.checked = true));
updateSelectedColumns();
});
noneBtn.addEventListener('click', e => {
e.preventDefault();
ul.querySelectorAll('input').forEach(cb => (cb.checked = false));
updateSelectedColumns();
});
div.appendChild(ul);
groupsEl.appendChild(div);
});
document.getElementById('columns_all').addEventListener('click', e => {
e.preventDefault();
groupsEl.querySelectorAll('input').forEach(cb => (cb.checked = true));
updateSelectedColumns();
});
document.getElementById('columns_none').addEventListener('click', e => {
e.preventDefault();
groupsEl.querySelectorAll('input').forEach(cb => (cb.checked = false));
updateSelectedColumns();
});
updateSelectedColumns();
bash-5.2#I’ll search for the updateSelectedColumns after reviewing the function isIntegerColumn at the top.
bash-5.2# grep -n "updateSelectedColumns" -n scubaduck/static/index.html
502: updateSelectedColumns();
617: cb.addEventListener('change', updateSelectedColumns);
626: updateSelectedColumns();
631: updateSelectedColumns();
639: updateSelectedColumns();
644: updateSelectedColumns();
646: updateSelectedColumns();
712:function updateSelectedColumns() {
918: updateSelectedColumns();
955: updateSelectedColumns();
1052: updateSelectedColumns();bash-5.2# sed -n '700,740p' scubaduck/static/index.html
const box = opt.closest('.rel-box');
const input = box.querySelector('input');
input.value = opt.dataset.value || opt.textContent;
opt.parentElement.style.display = 'none';
});
});
document.addEventListener('click', e => {
document.querySelectorAll('.rel-dropdown').forEach(dd => {
if (!dd.parentElement.contains(e.target)) dd.style.display = 'none';
});
});
function updateSelectedColumns() {
const base = allColumns.filter(name => {
const cb = document.querySelector(`#column_groups input[value="${name}"]`);
if (!cb || !cb.checked) return false;
if (graphTypeSel.value === 'table' && isStringColumn(name)) return false;
return true;
});
if (graphTypeSel.value === 'table' || graphTypeSel.value === 'timeseries') {
selectedColumns = groupBy.chips.slice();
if (document.getElementById('show_hits').checked) selectedColumns.push('Hits
');
base.forEach(c => {
if (!selectedColumns.includes(c)) selectedColumns.push(c);
});
derivedColumns.forEach(dc => {
if (dc.include && !selectedColumns.includes(dc.name)) selectedColumns.push
(dc.name);
});
} else {
selectedColumns = base.slice();
derivedColumns.forEach(dc => {
if (dc.include) selectedColumns.push(dc.name);
});
}
}
function isStringColumn(name) {
const t = (columnTypes[name] || '').toUpperCase();
return t.includes('CHAR') || t.includes('STRING') || t.includes('VARCHAR');
}
bash-5.2#I’m scrolling through the 902 section, where the time columns are resetting.
bash-5.2# sed -n '896,932p' scubaduck/static/index.html
}
function refreshDerivedColumns() {
allColumns.splice(0, allColumns.length, ...baseColumns);
stringColumns.splice(0, stringColumns.length, ...baseStringColumns);
integerColumns.splice(0, integerColumns.length, ...baseIntegerColumns);
timeColumns.splice(0, timeColumns.length, ...baseTimeColumns);
timeColumnOptions.splice(0, timeColumnOptions.length, ...baseTimeColumnOptions
);
derivedColumns.forEach(d => {
d.type = d.el.querySelector('.d-type').value;
d.name = d.el.querySelector('.d-name').value;
d.expr = d.el.querySelector('.d-expr').value;
d.include = d.el.querySelector('.d-use').checked;
allColumns.push(d.name);
columnTypes[d.name] = d.type;
if (d.type === 'string') {
stringColumns.push(d.name);
} else {
integerColumns.push(d.name);
timeColumnOptions.push(d.name);
}
});
updateSelectedColumns();
}
let lastQueryTime = 0;
let queryStart = 0;
function dive(push=true) {
const params = collectParams();
if (push) {
history.pushState(params, '', paramsToSearch(params));
}
const payload = Object.assign({}, params);
const dcMap = {};
(params.derived_columns || []).forEach(d => {
if (d.include) dcMap[d.name] = d.expr;bash-5.2# sed -n '946,1006p' scubaduck/static/index.html
lastQueryTime = Math.round(performance.now() - queryStart);
showResults(data);
})
.catch(err => {
showError(err);
});
}
function collectParams() {
updateSelectedColumns();
const payload = {
table: document.getElementById('table').value,
time_column: document.getElementById('time_column').value,
start: document.getElementById('start').value,
end: document.getElementById('end').value,
order_by: document.getElementById('order_by').value,
order_dir: orderDir,
limit: parseInt(document.getElementById('limit').value, 10),
columns: selectedColumns.filter(c =>
c !== 'Hits' && !derivedColumns.some(dc => dc.name === c)
),
graph_type: graphTypeSel.value,
filters: Array.from(document.querySelectorAll('#filters .filter')).map(f =>
{
const chips = f.chips || [];
const op = f.querySelector('.f-op').value;
let value = null;
if (op !== 'empty' && op !== '!empty') {
value = chips.length === 0 ? null : (chips.length === 1 ? chips[0] : chi
ps);
}
const colSel = f.querySelector('.f-col select') || f.querySelector('.f-col
');
return {column: colSel.value, op, value};
}),
derived_columns: Array.from(document.querySelectorAll('#derived_list .derive
d')).map(d => ({
type: d.querySelector('.d-type').value,
name: d.querySelector('.d-name').value,
expr: d.querySelector('.d-expr').value,
include: d.querySelector('.d-use').checked,
}))
};
if (graphTypeSel.value === 'table' || graphTypeSel.value === 'timeseries') {
payload.group_by = groupBy.chips || [];
payload.aggregate = document.getElementById('aggregate').value;
payload.show_hits = document.getElementById('show_hits').checked;
}
if (graphTypeSel.value === 'timeseries') {
payload.x_axis = document.getElementById('x_axis').value;
payload.granularity = document.getElementById('granularity').value;
payload.fill = document.getElementById('fill').value;
}
return payload;
}
function paramsToSearch(params) {
const sp = new URLSearchParams();
if (params.table) sp.set('table', params.table);
if (params.time_column) sp.set('time_column', params.time_column);
if (params.start) sp.set('start', params.start);
if (params.end) sp.set('end', params.end);
if (params.order_by) sp.set('order_by', params.order_by);
if (params.order_dir) sp.set('order_dir', params.order_dir);
if (params.limit !== null && params.limit !== undefined) sp.set('limit', param
s.limit);bash-5.2# sed -n '1006,1064p' scubaduck/static/index.html
if (params.limit !== null && params.limit !== undefined) sp.set('limit', param
s.limit);
if (params.columns && params.columns.length) sp.set('columns', params.columns.
join(','));
if (params.filters && params.filters.length) sp.set('filters', JSON.stringify(
params.filters));
if (params.derived_columns && params.derived_columns.length) sp.set('derived_c
olumns', JSON.stringify(params.derived_columns));
if (params.graph_type) sp.set('graph_type', params.graph_type);
if (params.graph_type === 'table' || params.graph_type === 'timeseries') {
if (params.group_by && params.group_by.length) sp.set('group_by', params.gro
up_by.join(','));
if (params.aggregate) sp.set('aggregate', params.aggregate);
if (params.show_hits) sp.set('show_hits', '1');
}
if (params.graph_type === 'timeseries') {
if (params.x_axis) sp.set('x_axis', params.x_axis);
if (params.granularity) sp.set('granularity', params.granularity);
if (params.fill) sp.set('fill', params.fill);
}
const qs = sp.toString();
return qs ? '?' + qs : '';
}
function applyParams(params) {
if (params.table) document.getElementById('table').value = params.table;
document.getElementById('time_column').value = params.time_column || defaultTi
meColumn;
document.getElementById('start').value = params.start || '';
document.getElementById('end').value = params.end || '';
if (params.order_by) {
document.getElementById('order_by').value = params.order_by;
}
orderDir = params.order_dir || 'ASC';
updateOrderDirButton();
if (params.limit !== undefined && params.limit !== null) {
document.getElementById('limit').value = params.limit;
}
graphTypeSel.value = params.graph_type || 'samples';
updateDisplayTypeUI();
if (params.x_axis) document.getElementById('x_axis').value = params.x_axis;
if (params.granularity) document.getElementById('granularity').value = params.
granularity;
if (params.fill) document.getElementById('fill').value = params.fill;
if (params.group_by) {
groupBy.chips.splice(0, groupBy.chips.length, ...params.group_by);
groupBy.renderChips();
}
if (params.aggregate) document.getElementById('aggregate').value = params.aggr
egate;
document.getElementById('show_hits').checked = params.show_hits ?? true;
document.querySelectorAll('#column_groups input').forEach(cb => {
cb.checked = !params.columns || params.columns.includes(cb.value);
});
updateSelectedColumns();
const dlist = document.getElementById('derived_list');
dlist.innerHTML = '';
derivedColumns.splice(0, derivedColumns.length);
if (params.derived_columns && params.derived_columns.length) {
params.derived_columns.forEach(dc => addDerived(dc));
}
refreshDerivedColumns();
const list = document.getElementById('filter_list');
list.innerHTML = '';
if (params.filters && params.filters.length) {
params.filters.forEach(f => {
addFilter();bash-5.2# sed -n '1064,1118p' scubaduck/static/index.html
addFilter();
const el = list.lastElementChild;
const colSel = el.querySelector('.f-col select') || el.querySelector('.f-c
ol');
colSel.value = f.column;
colSel.dispatchEvent(new Event('change'));
el.querySelector('.f-op').value = f.op;
el.querySelector('.f-op').dispatchEvent(new Event('change'));
if (f.value !== null && f.op !== 'empty' && f.op !== '!empty') {
const values = Array.isArray(f.value) ? f.value : [f.value];
values.forEach(v => el.addChip(v));
el.renderChips();
}
});
} else {
addFilter();
}
}
function parseSearch() {
const sp = new URLSearchParams(window.location.search);
const params = {};
if (sp.has('table')) params.table = sp.get('table');
if (sp.has('time_column')) params.time_column = sp.get('time_column');
if (sp.has('start')) params.start = sp.get('start');
if (sp.has('end')) params.end = sp.get('end');
if (sp.has('order_by')) params.order_by = sp.get('order_by');
if (sp.has('order_dir')) params.order_dir = sp.get('order_dir');
if (sp.has('limit')) params.limit = parseInt(sp.get('limit'), 10);
if (sp.has('columns')) params.columns = sp.get('columns').split(',').filter(c
=> c);
if (sp.has('filters')) {
try { params.filters = JSON.parse(sp.get('filters')); } catch(e) { params.fi
lters = []; }
}
if (sp.has('graph_type')) params.graph_type = sp.get('graph_type');
if (sp.has('group_by')) params.group_by = sp.get('group_by').split(',').filter
(c => c);
if (sp.has('aggregate')) params.aggregate = sp.get('aggregate');
if (sp.has('show_hits')) params.show_hits = sp.get('show_hits') === '1';
if (sp.has('x_axis')) params.x_axis = sp.get('x_axis');
if (sp.has('granularity')) params.granularity = sp.get('granularity');
if (sp.has('fill')) params.fill = sp.get('fill');
if (sp.has('derived_columns')) {
try { params.derived_columns = JSON.parse(sp.get('derived_columns')); } catc
h(e) { params.derived_columns = []; }
}
return params;
}
function initFromUrl() {
const params = parseSearch();
history.replaceState(params, '', paramsToSearch(params));
applyParams(params);
dive(false);
}
window.addEventListener('popstate', e => {
const params = e.state || parseSearch();
applyParams(params);bash-5.2# sed -n '1118,1176p' scubaduck/static/index.html
applyParams(params);
dive(false);
});
let originalRows = [];
let sortState = {index: null, dir: null};
function renderTable(rows) {
const table = document.getElementById('results');
table.innerHTML = '';
if (rows.length === 0) return;
let hitsIndex = selectedColumns.indexOf('Hits');
let totalHits = 0;
if (hitsIndex !== -1) {
totalHits = rows.reduce((s, r) => s + Number(r[hitsIndex]), 0);
}
const header = document.createElement('tr');
selectedColumns.forEach((col, i) => {
const th = document.createElement('th');
let label = col;
if (
displayType === 'table' &&
col !== 'Hits' &&
!(groupBy.chips || []).includes(col)
) {
const agg = document.getElementById('aggregate').value.toLowerCase();
label += ` (${agg})`;
}
th.textContent = label;
th.dataset.index = i;
th.addEventListener('click', handleSort);
if (sortState.index === i) {
th.classList.add('sorted');
th.textContent =
label + (sortState.dir === 'desc' ? ' \u25BC' : ' \u25B2');
}
if (!isStringColumn(col)) th.style.textAlign = 'right';
header.appendChild(th);
});
table.appendChild(header);
rows.forEach(row => {
const tr = document.createElement('tr');
tr.addEventListener('click', () => {
const wasSelected = tr.classList.contains('selected');
document
.querySelectorAll('#results tr.selected')
.forEach(el => el.classList.remove('selected'));
if (!wasSelected) {
tr.classList.add('selected');
}
});
row.forEach((v, i) => {
const col = selectedColumns[i];
const td = document.createElement('td');
if (isTimeColumn(col)) {
let d;
const t = (columnTypes[col] || '').toUpperCase();
if (t.includes('TIMESTAMP') || t.includes('DATE') || t.includes('TIME'))
{
d = new Date(v);bash-5.2# sed -n '1176,1236p' scubaduck/static/index.html
d = new Date(v);
} else {
d = new Date(Number(v) * 1000);
}
td.textContent = d.toLocaleString('en-US', {
weekday: 'short',
month: 'short',
day: 'numeric',
year: 'numeric',
hour: 'numeric',
minute: 'numeric',
second: 'numeric',
hour12: true,
timeZoneName: 'short'
});
} else {
if (col === 'Hits') {
const pct = totalHits ? ((v / totalHits) * 100).toFixed(1) : '0';
td.textContent = `${formatNumber(v)} (${pct}%)`;
} else {
td.textContent = isStringColumn(col) ? v : formatNumber(v);
}
}
td.style.textAlign = isStringColumn(col) ? 'left' : 'right';
tr.appendChild(td);
});
table.appendChild(tr);
});
// ensure table does not overflow unless necessary
const view = document.getElementById('view');
if (table.scrollWidth <= view.clientWidth) {
table.style.width = '100%';
}
}
function handleSort(e) {
const idx = parseInt(e.target.dataset.index, 10);
if (sortState.index !== idx) {
sortState.index = idx;
sortState.dir = 'asc';
} else if (sortState.dir === 'asc') {
sortState.dir = 'desc';
} else if (sortState.dir === 'desc') {
sortState.index = null;
sortState.dir = null;
} else {
sortState.dir = 'asc';
}
let rows = originalRows.slice();
if (sortState.index !== null) {
rows.sort((a, b) => {
const va = a[sortState.index];
const vb = b[sortState.index];
if (va === vb) return 0;
if (sortState.dir === 'desc') return va < vb ? 1 : -1;
return va > vb ? 1 : -1;
});
}
renderTable(rows);
}
bash-5.2#I'll adjust isTimeColumn to support integer and time types, and modify conversions for time unit.
bash-5.2# grep -n "1000" -n scubaduck/static/index.html | head
1178: d = new Date(Number(v) * 1000);
bash-5.2#I’ll explore timeseries_chart.js to check if there’s any conversion of numeric to seconds.
bash-5.2# grep -n "epoch" -r scubaduck
scubaduck/server.py:211: return f"TIMESTAMP 'epoch' + INTERVAL '1 sec
ond' * {col}"
scubaduck/server.py:225: f"CAST(floor((epoch({xexpr}) - epoch(TIM
ESTAMP '{params.start}'))/{sec}) AS BIGINT)"
scubaduck/server.py:229: f"TIMESTAMP 'epoch' + INTERVAL '{sec} se
cond' * "
scubaduck/server.py:230: f"CAST(floor(epoch({xexpr})/{sec}) AS BI
GINT)"
scubaduck/server.py:252: "TIMESTAMP 'epoch' + INTERVAL '1
second' * "
scubaduck/server.py:253: f"CAST(avg(epoch({col})) AS BIGI
NT)"bash-5.2# grep -n "new Date" -n scubaduck/static/js/timeseries_chart.js
7: return new Date(s).getTime();
9: return new Date(s + 'Z').getTime();
176: let d = new Date(start);
197: const d = new Date(t);
419: text.textContent = fmt(new Date(t), lu);bash-5.2# grep -n "\* 1000" -n scubaduck/static/js/timeseries_chart.js
34: const bucketMs = (data.bucket_size || 3600) * 1000;
bash-5.2#I'm checking how times are parsed in showTimeSeries(), particularly around line 170 in timeseries_chart.js.
bash-5.2# sed -n '160,210p' scubaduck/static/js/timeseries_chart.js
let best = intervals[0];
let bestScore = Infinity;
intervals.forEach(i => {
const count = span / approxMs(i);
const score = Math.abs(count - 15);
if (score < bestScore) {
best = i;
bestScore = score;
}
});
return best;
}
function generateTicks(start, end, intv) {
const ticks = [];
if (intv.unit === 'month' || intv.unit === 'year') {
let d = new Date(start);
d.setUTCDate(1);
if (intv.unit === 'year') d.setUTCMonth(0);
let unitVal =
intv.unit === 'month'
? d.getUTCFullYear() * 12 + d.getUTCMonth()
: d.getUTCFullYear();
unitVal = Math.ceil(unitVal / intv.step) * intv.step;
while (true) {
const year =
intv.unit === 'month' ? Math.floor(unitVal / 12) : unitVal;
const month = intv.unit === 'month' ? unitVal % 12 : 0;
const t = Date.UTC(year, month, 1);
if (t > end) break;
if (t >= start) ticks.push(t);
unitVal += intv.step;
}
} else {
const step = intv.ms * intv.step;
let t = Math.ceil(start / step) * step;
if (intv.unit === 'week') {
const d = new Date(t);
const adj = (d.getUTCDay() + 6) % 7;
t = d.getTime() - adj * 86400000;
t = Math.ceil(t / step) * step;
}
if (t === start) t += step;
for (; t <= end; t += step) ticks.push(t);
}
return ticks;
}
function labelUnit(intv) {
if (intv.unit === 'year') return 'year';
if (intv.unit === 'month') return 'month';
bash-5.2#I'll modify the query and render logic around time units to handle conversions correctly.
I'll modify time column formatting logic to respect time unit selection without affecting isTimeColumn.
Time unit selection applies only to the selected time column. I'll implement related changes.
I'll implement the time expression logic for handling different time units and ensuring compatibility.
I'll add new tests for milliseconds by inspecting the existing integer_time_column test.
bash-5.2# grep -n "integer_time_column" -n tests/test_server.py
319:def test_integer_time_column(tmp_path: Path) -> None:bash-5.2# sed -n '318,344p' tests/test_server.py
def test_integer_time_column(tmp_path: Path) -> None:
csv_file = tmp_path / "events.csv"
csv_file.write_text("created,event\n1704067200,login\n1704070800,logout\n")
app = server.create_app(csv_file)
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-01 01:00:00",
"order_by": "created",
"columns": ["created", "event"],
"time_column": "created",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert len(data["rows"]) == 2
def test_envvar_db(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
csv_file = tmp_path / "custom.csv"
csv_file.write_text("timestamp,event,value,user\n2024-01-01 00:00:00,login,5
,bob\n")
monkeypatch.setenv("SCUBADUCK_DB", str(csv_file))
app = server.create_app()
bash-5.2#I’ll add a test for integer time with milliseconds and check the existing sample CSV.
bash-5.2# cat scubaduck/sample.csv | head
timestamp,event,value,user
2024-01-01 00:00:00,login,10,alice
2024-01-01 01:00:00,logout,20,bob
2024-01-02 00:00:00,login,30,alice
2024-01-02 03:00:00,login,40,charlie
bash-5.2#I'll check test_web.py for time-related tests and time_unit usage to see if updates are needed.
bash-5.2# grep -n "derived" -n tests/test_web.py
339: expr = page.query_selector("#derived_list .derived textarea")
341: name_inp = page.query_selector("#derived_list .derived .d-name")
1012:def test_derived_column_query(page: Any, server_url: str) -> None:
1017: expr = page.query_selector("#derived_list .derived textarea")
1031:def test_derived_column_remove(page: Any, server_url: str) -> None:
1036: assert page.query_selector("#derived_list .derived button.remove")
1037: page.click("#derived_list .derived button.remove")
1038: count = page.evaluate("document.querySelectorAll('#derived_list .derive
d').length")60: order_by="timestamp",
66: timestamps = [parser.parse(row[0]).replace(tzinfo=None) for row in data["
rows"]]
67: assert timestamps == [
93: order_by="timestamp",
99:def test_time_column_dropdown(page: Any, server_url: str) -> None:
101: page.wait_for_selector("#time_column option", state="attached")
102: options = page.locator("#time_column option").all_inner_texts()
103: assert "timestamp" in options
105: assert page.input_value("#time_column") == "timestamp"
166: assert "timestamp" in cols
bash-5.2#The user needs time unit selection added in the UI; I'll review tests/test_web.
select_value(page, "#graph_type", "timeseries")
page.fill("#start", "2024-01-01 00:00:00")
page.fill("#end", "2024-01-03 12:00:00")
select_value(page, "#granularity", "1 hour")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
page.wait_for_selector("#chart text.tick-label", state="attached")
labels = page.eval_on_selector_all(
"#chart text.tick-label", "els => els.map(e => e.textContent)"
)
assert labels
assert all(lbl != "00:00" for lbl in labels)
times = [lbl for lbl in labels if ":" in lbl]
assert times
for t in times:
h = int(t.split(":")[0])
assert h % 4 == 0
def test_timeseries_legend_values(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#graph_type", state="attached")
select_value(page, "#graph_type", "timeseries")
page.evaluate("g => { groupBy.chips = g; groupBy.renderChips(); }", ["user"]
)
select_value(page, "#aggregate", "Avg")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
headers = page.evaluate(
"() => Array.from(document.querySelectorAll('#legend .legend-header')).m
ap(e => e.textContent)"
)
assert any(h.startswith("alice") for h in headers)
page.wait_for_selector("#chart path", state="attached")
page.eval_on_selector(
"#chart",
"el => { const r=el.getBoundingClientRect(); el.dispatchEvent(new MouseE
vent('mousemove', {clientX:r.left+r.width/2, clientY:r.top+r.height/2, bubbles:t
rue})); }",
)
value = page.evaluate("document.querySelector('#legend .legend-value').textC
ontent")
assert value != ""
bash-5.2#I’ll add a test for the time unit, specifically checking backend behavior for different units.
I’ll modify _time_expr to handle the unit parameter and check whether nanoseconds are supported.
I’ll adjust the multiplier formula to avoid decimals and ensure proper fractional handling.
I’ll format decimals to avoid exponentials, ensuring consistent precision for nanoseconds handling.
return max(int(total // buckets), 1)
return 3600
def _time_expr(col: str, column_types: Dict[str, str] | None) -> str:
"""Return SQL expression for column interpreted as timestamp."""
if column_types is None:
return col
ctype = column_types.get(col, "").upper()
if not any(t in ctype for t in ["TIMESTAMP", "DATE", "TIME"]):
if any(
t in ctype
for t in [
"INT",
"DECIMAL",
"REAL",
"DOUBLE",
"FLOAT",
"NUMERIC",
"HUGEINT",
]
):
return f"TIMESTAMP 'epoch' + INTERVAL '1 second' * {col}"
return col
def build_query(params: QueryParams, column_types: Dict[str, str] | None = None)
-> str:
select_parts: list[str] = []
group_cols = params.group_by[:]
if params.graph_type == "timeseries":
sec = _granularity_seconds(params.granularity, params.start, params.end)
x_axis = params.x_axis or params.time_column
xexpr = _time_expr(x_axis, column_types)
if params.start:
bucket_expr = (
f"TIMESTAMP '{params.start}' + INTERVAL '{sec} second' * "
f"CAST(floor((epoch({xexpr}) - epoch(TIMESTAMP '{params.start}')
)/{sec}) AS BIGINT)"
)
else:
bucket_expr = (
f"TIMESTAMP 'epoch' + INTERVAL '{sec} second' * "
f"CAST(floor(epoch({xexpr})/{sec}) AS BIGINT)"
)
select_parts.append(f"{bucket_expr} AS bucket")
group_cols = ["bucket"] + group_cols
has_agg = bool(group_cols) or params.aggregate is not None
if has_agg:
select_cols = (
group_cols[1:] if params.graph_type == "timeseries" else group_cols
)
select_parts.extend(select_cols)
agg = (params.aggregate or "avg").lower()
def agg_expr(col: str) -> str:
if agg.startswith("p"):
quant = float(agg[1:]) / 100
return f"quantile({col}, {quant})"
if agg == "count distinct":
return f"count(DISTINCT {col})"
if agg == "avg" and column_types is not None:
ctype = column_types.get(col, "").upper()
if "TIMESTAMP" in ctype or "DATE" in ctype or "TIME" in ctype:
return (
"TIMESTAMP 'epoch' + INTERVAL '1 second' * "
f"CAST(avg(epoch({col})) AS BIGINT)"
)
return f"{agg}({col})"
for col in params.columns:
if col in group_cols:
continue
select_parts.append(f"{agg_expr(col)} AS {col}")
if params.show_hits:
select_parts.insert(len(group_cols), "count(*) AS Hits")
else:
select_parts.extend(params.columns)
if has_agg and params.derived_columns:
inner_params = replace(
params,
derived_columns={},
order_by=None,
limit=None,
)
inner_sql = build_query(inner_params, column_types)
outer_select = ["t.*"] + [
f"{expr} AS {name}" for name, expr in params.derived_columns.items()
]
indented_inner = "\n".join(" " + line for line in inner_sql.splitline
s())
lines = [
f"SELECT {', '.join(outer_select)}",
"FROM (",
indented_inner,
") t",
]
if params.order_by:
lines.append(f"ORDER BY {params.order_by} {params.order_dir}")
elif params.graph_type == "timeseries":
lines.append("ORDER BY bucket")
if params.limit is not None:
lines.append(f"LIMIT {params.limit}")
return "\n".join(lines)
for name, expr in params.derived_columns.items():
select_parts.append(f"{expr} AS {name}")
select_clause = ", ".join(select_parts) if select_parts else "*"
lines = [f"SELECT {select_clause}", f'FROM "{params.table}"']
where_parts: list[str] = []
time_expr = _time_expr(params.time_column, column_types)
if params.start:
where_parts.append(f"{time_expr} >= '{params.start}'")
if params.end:
where_parts.append(f"{time_expr} <= '{params.end}'")
for f in params.filters:
op = f.op
if op in {"empty", "!empty"}:
val = "''"
else:
if f.value is None:
continue
bash-5.2#I'll view additional lines for the _time_expr call to gather more context.
for name, expr in params.derived_columns.items():
select_parts.append(f"{expr} AS {name}")
select_clause = ", ".join(select_parts) if select_parts else "*"
lines = [f"SELECT {select_clause}", f'FROM "{params.table}"']
where_parts: list[str] = []
time_expr = _time_expr(params.time_column, column_types)
if params.start:
where_parts.append(f"{time_expr} >= '{params.start}'")
if params.end:
where_parts.append(f"{time_expr} <= '{params.end}'")
for f in params.filters:
op = f.op
if op in {"empty", "!empty"}:
val = "''"
else:
if f.value is None:
continue
if isinstance(f.value, list):
if not f.value:
continue
if op == "=":
vals = " OR ".join(f"{f.column} = '{v}'" for v in f.value)
where_parts.append(f"({vals})")
continue
val = f"'{f.value}'" if isinstance(f.value, str) else str(f.value)
if op == "contains":
where_parts.append(f"{f.column} ILIKE '%' || {val} || '%'")
elif op == "!contains":
where_parts.append(f"{f.column} NOT ILIKE '%' || {val} || '%'")
elif op == "empty":
where_parts.append(f"{f.column} = {val}")
elif op == "!empty":
where_parts.append(f"{f.column} != {val}")
else:
where_parts.append(f"{f.column} {op} {val}")
if where_parts:
lines.append("WHERE " + " AND ".join(where_parts))
if group_cols:
lines.append("GROUP BY " + ", ".join(group_cols))
if params.order_by:
lines.append(f"ORDER BY {params.order_by} {params.order_dir}")
elif params.graph_type == "timeseries":
lines.append("ORDER BY bucket")
if params.limit is not None:
lines.append(f"LIMIT {params.limit}")
return "\n".join(lines)
def create_app(db_file: str | Path | None = None) -> Flask:
app = Flask(__name__, static_folder="static")
if db_file is None:
env_db = os.environ.get("SCUBADUCK_DB")
if env_db:
db_file = env_db
db_path = Path(db_file or Path(__file__).with_name("sample.csv")).resolve()
con = _load_database(db_path)
tables = [r[0] for r in con.execute("SHOW TABLES").fetchall()]
if not tables:
raise ValueError("No tables found in database")
default_table = tables[0]
columns_cache: Dict[str, Dict[str, str]] = {}
def get_columns(table: str) -> Dict[str, str]:
if table not in columns_cache:
rows = con.execute(f'PRAGMA table_info("{table}")').fetchall()
if not rows:
raise ValueError(f"Unknown table: {table}")
columns_cache[table] = {r[1]: r[2] for r in rows}
return columns_cache[table]
sample_cache: Dict[Tuple[str, str, str], Tuple[List[str], float]] = {}
CACHE_TTL = 60.0
CACHE_LIMIT = 200
@app.route("/")
def index() -> Any: # pyright: ignore[reportUnusedFunction]
assert app.static_folder is not None
return send_from_directory(app.static_folder, "index.html")
@app.route("/js/<path:filename>")
def js(filename: str) -> Any: # pyright: ignore[reportUnusedFunction]
assert app.static_folder is not None
folder = Path(app.static_folder) / "js"
return send_from_directory(folder, filename)
@app.route("/api/tables")
def tables_endpoint() -> Any: # pyright: ignore[reportUnusedFunction]
return jsonify(tables)
bash-5.2#I'll continue scrolling to reach the specific query route and confirm the details.
bash-5.2# sed -n '432,500p' scubaduck/server.py
start = parse_time(payload.get("start"))
end = parse_time(payload.get("end"))
except Exception as exc:
return jsonify({"error": str(exc)}), 400
params = QueryParams(
start=start,
end=end,
order_by=payload.get("order_by"),
order_dir=payload.get("order_dir", "ASC"),
limit=payload.get("limit"),
columns=payload.get("columns", []),
derived_columns=payload.get("derived_columns", {}),
graph_type=payload.get("graph_type", "samples"),
group_by=payload.get("group_by", []),
aggregate=payload.get("aggregate"),
show_hits=payload.get("show_hits", False),
x_axis=payload.get("x_axis"),
granularity=payload.get("granularity", "Auto"),
fill=payload.get("fill", "0"),
table=payload.get("table", default_table),
time_column=payload.get("time_column", "timestamp"),
)
for f in payload.get("filters", []):
params.filters.append(Filter(f["column"], f["op"], f.get("value")))
if params.table not in tables:
return jsonify({"error": "Invalid table"}), 400
column_types = get_columns(params.table)
if params.time_column not in column_types:
return jsonify({"error": "Invalid time_column"}), 400
if params.graph_type not in {"table", "timeseries"} and (
params.group_by or params.aggregate or params.show_hits
):
return (
jsonify(
{
"error": "group_by, aggregate and show_hits are only val
id for table or timeseries view"
}
),
400,
)
valid_cols = set(column_types.keys())
valid_cols.update(params.derived_columns.keys())
if params.graph_type == "timeseries":
if params.x_axis is None:
params.x_axis = params.time_column
if params.x_axis not in valid_cols:
return jsonify({"error": "Invalid x_axis"}), 400
ctype = column_types.get(params.x_axis, "").upper()
is_time = any(t in ctype for t in ["TIMESTAMP", "DATE", "TIME"])
is_numeric = any(
t in ctype
for t in [
"INT",
"DECIMAL",
"REAL",
"DOUBLE",
"FLOAT",
"NUMERIC",
"HUGEINT",
]
)
if not (is_time or is_numeric):
return jsonify({"error": "x_axis must be a time column"}), 400
bash-5.2#I'm checking the function's end and considering parsing 'time_unit' from the query.
bash-5.2# grep -n "time_unit" -n scubaduck
grep: scubaduck: Is a directory
bash-5.2#I’ll revisit tests/test_web to verify the query string parsing, searching for 'params_to_search'.
bash-5.2# grep -n "params_to_search" -r scubaduck
bash-5.2#I'll review the functions in test_web, focusing on "params" or "parse_search."
bash-5.2# grep -n "Search" -n tests/test_web.py
bash-5.2#I'll adjust functions like parseSearch, applyParams, and collectParams to support time_unit.
bash-5.2# grep -n "timeColumnSelect" -n scubaduck/static/index.html
518: const timeColumnSelect = document.getElementById('time_column');
521: timeColumnSelect.innerHTML = '';
586: timeColumnSelect.appendChild(o);
bash-5.2#I’m looking for the 'time_column' event to check its usage.
bash-5.2# grep -n "time_column'" -n scubaduck/static/index.html
518: const timeColumnSelect = document.getElementById('time_column');
749: const sel = document.getElementById('time_column').value;
958: time_column: document.getElementById('time_column').value,
1001: if (params.time_column) sp.set('time_column', params.time_column);
1027: document.getElementById('time_column').value = params.time_column || defa
ultTimeColumn;
1086: if (sp.has('time_column')) params.time_column = sp.get('time_column');
bash-5.2#I’ll just add the new <select> for the time unit next to the time column, keeping it visible. I'll add the necessary CSS for styling. Looking at index.html for guidance will help me match the style.
bash-5.2# grep -n "field" -n scubaduck/static/index.html | head
13: .field { display: flex; align-items: center; margin-bottom: 10px; }
14: .field label { width: 80px; text-align: right; margin-right: 5px; }
193: <div class="field">
197: <div class="field">
215: <div class="field">
234: <div class="field">
239: <div class="field">
243: <div id="x_axis_field" class="field" style="display:none;">
247: <div id="granularity_field" class="field" style="display:none;">
270: <div id="fill_field" class="field" style="display:none;">bash-5.2# sed -n '1,20p' scubaduck/static/index.html
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<title>ScubaDuck</title>
<style>
body { margin: 0; font-family: sans-serif; height: 100vh; display: flex; fle
x-direction: column; }
#header { padding: 10px; font-weight: bold; border-bottom: 1px solid #ccc; }
#content { flex: 1; display: flex; height: calc(100vh - 42px); overflow: hid
den; }
#sidebar { width: 450px; padding: 10px; border-right: 3px solid #ccc; overfl
ow-y: auto; display: flex; flex-direction: column; box-sizing: border-box; }
#sidebar-resizer { width: 5px; cursor: col-resize; background: #ccc; }
#view { flex: 1; padding: 10px; overflow-y: auto; overflow-x: auto; }
.field { display: flex; align-items: center; margin-bottom: 10px; }
.field label { width: 80px; text-align: right; margin-right: 5px; }
.help { margin-left: 4px; cursor: help; }
.rel-btn { margin-left: 4px; }
#tabs { display: flex; align-items: center; margin-bottom: 10px; }
#tabs .tab { margin-right: 5px; background: none; border: 1px solid #ccc; pa
dding: 4px 8px; cursor: pointer; width: 120px; text-align: center; box-sizing: b
order-box; }
#tabs .tab.active { background: #eee; font-weight: bold; }
#dive { margin-left: auto; background: green; color: white; border: none; pa
dding: 5px 10px; cursor: pointer; }
bash-5.2#I’ll add a margin-left for styling and look for any width-customizing instructions.
bash-5.2# sed -n '20,60p' scubaduck/static/index.html
#dive { margin-left: auto; background: green; color: white; border: none; pa
dding: 5px 10px; cursor: pointer; }
.tab-content { display: none; }
.tab-content.active { display: block; }
#filter_list { display: flex; flex-direction: column; }
#filters .filter {
border: 1px solid #ccc;
padding: 5px;
margin-bottom: 5px;
position: relative;
display: flex;
flex-direction: column;
}
#derived_columns .derived {
border: 1px solid #ccc;
padding: 5px;
margin-bottom: 5px;
display: flex;
flex-direction: column;
}
#derived_columns .derived-row {
display: flex;
margin-bottom: 5px;
}
#derived_columns .derived-row input[type="text"] {
margin-left: 5px;
flex: 1;
}
#derived_columns .derived-row button.remove {
margin-left: 5px;
width: 20px;
flex: 0 0 auto;
padding: 0;
text-align: center;
line-height: 1;
}
#derived_columns textarea {
width: 100%;
box-sizing: border-box;
}
#filters .filter-row { display: flex; margin-bottom: 5px; }
#filters .filter-row .f-col { flex: 1; }
bash-5.2#I’m continuing to scroll to capture more details.
bash-5.2# sed -n '60,120p' scubaduck/static/index.html
#filters .filter-row .f-col { flex: 1; }
#filters .filter-row .f-op {
margin-left: 5px;
width: fit-content;
flex: 0 0 auto;
}
.chip-input input {
border: none;
flex: 1;
min-width: 60px;
margin: 2px;
outline: none;
}
.chip-box { position: relative; }
.chip-input { display: flex; flex-wrap: wrap; border: 1px solid #ccc; paddin
g: 2px; min-height: 24px; }
.chip { background: #eee; border: 1px solid #999; padding: 2px 4px; margin:
2px; border-radius: 3px; display: flex; align-items: center; }
.chip .x { margin-left: 4px; cursor: pointer; }
.chip-copy { margin-left: 4px; cursor: pointer; background: none; border: no
ne; }
.chip-dropdown { position: absolute; left: 0; right: 0; top: 100%; backgroun
d: white; border: 1px solid #ccc; max-height: 120px; overflow-y: auto; z-index:
10; display: none; }
.chip-dropdown div { padding: 2px 4px; cursor: pointer; }
.chip-dropdown div.highlight { background: #bde4ff; }
.rel-box { position: relative; display: flex; }
.rel-dropdown { position: absolute; left: 0; right: 0; top: 100%; background
: white; border: 1px solid #ccc; z-index: 10; display: none; }
.rel-dropdown div { padding: 2px 4px; cursor: pointer; }
.rel-dropdown div:hover { background: #bde4ff; }
.dropdown { position: relative; display: inline-block; }
.dropdown-display {
border: 1px solid #ccc;
padding: 2px 18px 2px 4px;
cursor: pointer;
min-width: 80px;
position: relative;
}
.dropdown-display::after {
content: '\25BC';
position: absolute;
right: 4px;
pointer-events: none;
}
.dropdown-menu { position: absolute; left: 0; right: 0; top: 100%; backgroun
d: white; border: 1px solid #ccc; z-index: 10; max-height: 160px; overflow-y: au
to; display: none; }
.dropdown-menu input { width: 100%; box-sizing: border-box; padding: 2px 4px
; border: none; border-bottom: 1px solid #ccc; }
.dropdown-menu div { padding: 2px 4px; cursor: pointer; }
.dropdown-menu div.selected { background: #bde4ff; }
.dropdown-menu .option:hover { background: #eee; }
.dropdown-menu input::placeholder { color: #999; }
#filters .filter button.remove {
margin-left: 5px;
width: 20px;
flex: 0 0 auto;
padding: 0;
text-align: center;
line-height: 1;
}
#filters h4 { margin: 0 0 5px 0; }
table { border-collapse: collapse; min-width: 100%; }
th, td { border: 1px solid #ccc; padding: 4px; box-sizing: border-box; }
th { text-align: left; cursor: pointer; position: relative; }
th.sorted { color: blue; }
tr:nth-child(even) td { background: #f9f9f9; }
tr.selected td { background: #bde4ff !important; }
tr:hover:not(.selected) td { background: #eee; }
bash-5.2#I'll search for references to initDropdown to check how selects are handled.
bash-5.2# grep -n "initDropdown" -n scubaduck/static/index.html
358:function initDropdown(select) {
651: initDropdown(orderSelect);
652: initDropdown(document.getElementById('aggregate'));
797: initDropdown(colSel);bash-5.2# sed -n '646,664p' scubaduck/static/index.html
updateSelectedColumns();
groupBy = document.getElementById('group_by').closest('.field');
initChipInput(groupBy, typed =>
allColumns.filter(c => c.toLowerCase().includes(typed.toLowerCase()))
);
initDropdown(orderSelect);
initDropdown(document.getElementById('aggregate'));
});
}
let columnsInitialized = false;
fetch('/api/tables').then(r => r.json()).then(tables => {
tables.forEach(t => {
const o = document.createElement('option');
o.value = t;
o.textContent = t;
document.getElementById('table').appendChild(o);
});
const table = parseSearch().table || tables[0];
bash-5.2#I'll scroll past line 797 to check other references in the table code.
bash-5.2# sed -n '792,808p' scubaduck/static/index.html
<div class="chip-dropdown"></div>
</div>
`;
const colSel = container.querySelector('.f-col');
colSel.innerHTML = allColumns.map(c => `<option value="${c}">${c}</option>`).j
oin('');
initDropdown(colSel);
function populateOps() {
const opSel = container.querySelector('.f-op');
const col = colSel.value;
const ops = isStringColumn(col)
? [
['=', '='],
['!=', '!='],
['~', 'matches regex'],
['!~', 'not matches regex'],
['contains', 'contains'],
bash-5.2#I'm checking the loadColumns function; it might need initDropdown for time_column or x_axis.
bash-5.2# grep -n "time_column" -n -n scubaduck/static/index.html | head
195: <select id="time_column"></select>
518: const timeColumnSelect = document.getElementById('time_column');
749: const sel = document.getElementById('time_column').value;
958: time_column: document.getElementById('time_column').value,
1001: if (params.time_column) sp.set('time_column', params.time_column);
1027: document.getElementById('time_column').value = params.time_column || defa
ultTimeColumn;
1086: if (sp.has('time_column')) params.time_column = sp.get('time_column');
bash-5.2#I'm checking if initDropdown is called after timeColumnSelect in the loadColumns function.
bash-5.2# grep -n "initDropdown(orderSelect" -n scubaduck/static/index.html
651: initDropdown(orderSelect);bash-5.2# sed -n '632,660p' scubaduck/static/index.html
});
div.appendChild(ul);
groupsEl.appendChild(div);
});
document.getElementById('columns_all').addEventListener('click', e => {
e.preventDefault();
groupsEl.querySelectorAll('input').forEach(cb => (cb.checked = true));
updateSelectedColumns();
});
document.getElementById('columns_none').addEventListener('click', e => {
e.preventDefault();
groupsEl.querySelectorAll('input').forEach(cb => (cb.checked = false));
updateSelectedColumns();
});
updateSelectedColumns();
groupBy = document.getElementById('group_by').closest('.field');
initChipInput(groupBy, typed =>
allColumns.filter(c => c.toLowerCase().includes(typed.toLowerCase()))
);
initDropdown(orderSelect);
initDropdown(document.getElementById('aggregate'));
});
}
let columnsInitialized = false;
fetch('/api/tables').then(r => r.json()).then(tables => {
tables.forEach(t => {
const o = document.createElement('option');
o.value = t;
bash-5.2#Not all selects are turned into dropdowns. Time_column uses a plain <select>, so I'll leave time_unit as plain too and toggle visibility based on column type. I'll check similar fields for this behavior.
bash-5.2# grep -n "x_axis_field" -n scubaduck/static/index.html
243: <div id="x_axis_field" class="field" style="display:none;">
483: document.getElementById('x_axis_field').style.display = showTS ? 'flex' :
'none';
bash-5.2#I’ll take a look around line 480 to see how things are hidden.
bash-5.2# sed -n '468,500p' scubaduck/static/index.html
}
sidebarResizer.addEventListener('mousedown', startSidebarDrag);
let orderDir = 'ASC';
const orderDirBtn = document.getElementById('order_dir');
const graphTypeSel = document.getElementById('graph_type');
function updateOrderDirButton() {
orderDirBtn.textContent = orderDir + (orderDir === 'ASC' ? ' \u25B2' : ' \u25B
C');
}
function updateDisplayTypeUI() {
const showTable = graphTypeSel.value === 'table';
const showTS = graphTypeSel.value === 'timeseries';
document.getElementById('group_by_field').style.display = showTable || showTS
? 'flex' : 'none';
document.getElementById('aggregate_field').style.display = showTable || showTS
? 'flex' : 'none';
document.getElementById('show_hits_field').style.display = showTable ? 'flex'
: 'none';
document.getElementById('x_axis_field').style.display = showTS ? 'flex' : 'non
e';
document.getElementById('granularity_field').style.display = showTS ? 'flex' :
'none';
document.getElementById('fill_field').style.display = showTS ? 'flex' : 'none'
;
document.querySelectorAll('#column_groups .col-group').forEach(g => {
if (g.querySelector('.col-group-header').textContent.startsWith('Strings'))
{
g.style.display = showTable || showTS ? 'none' : '';
}
});
if (showTS) {
const lim = document.getElementById('limit');
if (!lim.dataset.setByUser) {
lim.value = '7';
}
document.querySelectorAll('#column_groups input').forEach(cb => {
if (isTimeColumn(cb.value) || isStringColumn(cb.value)) {
cb.checked = false;
}
});
bash-5.2#I'll continue scrolling to capture more information.
bash-5.2# sed -n '500,520p' scubaduck/static/index.html
});
document.getElementById('order_by').value = '';
updateSelectedColumns();
}
displayType = graphTypeSel.value;
}
orderDirBtn.addEventListener('click', () => {
orderDir = orderDir === 'ASC' ? 'DESC' : 'ASC';
updateOrderDirButton();
});
updateOrderDirButton();
graphTypeSel.addEventListener('change', updateDisplayTypeUI);
function loadColumns(table) {
return fetch('/api/columns?table=' + encodeURIComponent(table)).then(r => r.js
on()).then(cols => {
const orderSelect = document.getElementById('order_by');
const xAxisSelect = document.getElementById('x_axis');
const groupsEl = document.getElementById('column_groups');
const timeColumnSelect = document.getElementById('time_column');
orderSelect.innerHTML = '';
xAxisSelect.innerHTML = '';
bash-5.2#I'll add a 'time_unit' select after the time_column. It will show only if the column is an integer.
bash-5.2# grep -n "#time_column" -n scubaduck/static/index.html
bash-5.2#No events needed; I'll make the dropdown visible, update JS to handle time unit parameters. Then, modify server and tests.
bash-5.2# grep -n "create_app" -n tests/test_server.py | head
199: app = server.create_app(db)
248: app = server.create_app(sqlite_file)
304: app = server.create_app(sqlite_file)
322: app = server.create_app(csv_file)
344: app = server.create_app()
358: server.create_app()
bash-5.2#I'll inspect lines near 248 for test examples to match the style.
def __getattr__(self, name: str) -> object:
return getattr(self.con, name)
return Wrapper(real)
monkeypatch.setattr(server.duckdb, "connect", failing_connect)
app = server.create_app(sqlite_file)
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-01 01:00:00",
"order_by": "timestamp",
"columns": ["timestamp", "url", "title"],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert data["rows"][0][1] == "https://a.com"
def test_sqlite_bigint(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
sqlite_file = tmp_path / "big.sqlite"
import sqlite3
conn = sqlite3.connect(sqlite_file)
conn.execute("CREATE TABLE events (timestamp TEXT, value INTEGER)")
big_value = 13385262862605259
conn.execute(
"INSERT INTO events VALUES ('2024-01-01 00:00:00', ?)",
(big_value,),
)
conn.commit()
conn.close() # pyright: ignore[reportUnknownMemberType, reportAttributeAcce
ssIssue]
from typing import Any
real_connect = duckdb.connect
def failing_connect(*args: Any, **kwargs: Any) -> Any:
real = real_connect(*args, **kwargs)
class Wrapper:
def __init__(self, con: duckdb.DuckDBPyConnection) -> None:
self.con = con
self._failed = False
def execute(self, sql: str, *a: Any, **kw: Any):
if not self._failed and sql == "LOAD sqlite":
self._failed = True
raise RuntimeError("fail")
return self.con.execute(sql, *a, **kw)
def __getattr__(self, name: str) -> object:
return getattr(self.con, name)
return Wrapper(real)
monkeypatch.setattr(server.duckdb, "connect", failing_connect)
app = server.create_app(sqlite_file)
client = app.test_client()
payload = {
"table": "events",
"order_by": "timestamp",
"columns": ["timestamp", "value"],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert data["rows"][0][1] == big_value
def test_integer_time_column(tmp_path: Path) -> None:
csv_file = tmp_path / "events.csv"
csv_file.write_text("created,event\n1704067200,login\n1704070800,logout\n")
app = server.create_app(csv_file)
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-01 01:00:00",
"order_by": "created",
bash-5.2#I need to check a bit more of the test to catch the full context.
"order_by": "created",
"columns": ["created", "event"],
"time_column": "created",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert len(data["rows"]) == 2
def test_envvar_db(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
csv_file = tmp_path / "custom.csv"
csv_file.write_text("timestamp,event,value,user\n2024-01-01 00:00:00,login,5
,bob\n")
monkeypatch.setenv("SCUBADUCK_DB", str(csv_file))
app = server.create_app()
client = app.test_client()
payload = _make_payload()
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
rows = rv.get_json()["rows"]
assert len(rows) == 1
def test_envvar_db_missing(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> N
one:
missing = tmp_path / "missing.sqlite"
monkeypatch.setenv("SCUBADUCK_DB", str(missing))
bash-5.2#I'll add the new test, test_integer_time_column_ms, after test_integer_time_column. Let's review the full file.
def test_timeseries_derived_column() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "timeseries",
"granularity": "1 hour",
"limit": 7,
"columns": ["value"],
"derived_columns": {"derived_1": "value * 2"},
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
rows = data["rows"]
assert all(r[2] == r[1] * 2 for r in rows)
def test_default_start_end_returned() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"order_by": "timestamp",
"limit": 5,
"columns": ["timestamp"],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert data["start"] == "2024-01-01 00:00:00"
assert data["end"] == "2024-01-02 03:00:00"
bash-5.2#I’m planning to add a test for test_integer_time_column_ms and modify tests accordingly.
from typing import Any
def select_value(page: Any, selector: str, value: str) -> None:
page.evaluate(
"arg => setSelectValue(arg.sel, arg.val)",
{"sel": selector, "val": value},
)
def run_query(
page: Any,
url: str,
*,
start: str | None = None,
end: str | None = None,
order_by: str | None = None,
order_dir: str | None = "ASC",
limit: int | None = None,
group_by: list[str] | None = None,
aggregate: str | None = None,
) -> dict[str, Any]:
page.goto(url)
page.wait_for_selector("#order_by option", state="attached")
page.wait_for_selector("#order_dir", state="attached")
page.wait_for_function("window.lastResults !== undefined")
if start is not None:
page.fill("#start", start)
if end is not None:
page.fill("#end", end)
if order_by is not None:
select_value(page, "#order_by", order_by)
if order_dir is not None and order_dir == "DESC":
page.click("#order_dir")
if limit is not None:
page.fill("#limit", str(limit))
if group_by is not None:
select_value(page, "#graph_type", "table")
bash-5.2#I’m scrolling further to review the remaining functions.
page.evaluate(
"g => { groupBy.chips = g; groupBy.renderChips(); }",
group_by,
)
if aggregate is not None:
select_value(page, "#graph_type", "table")
select_value(page, "#aggregate", aggregate)
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
return page.evaluate("window.lastResults")
def test_range_filters(page: Any, server_url: str) -> None:
data = run_query(
page,
server_url,
start="2024-01-02 00:00:00",
end="2024-01-02 04:00:00",
order_by="timestamp",
limit=100,
)
assert len(data["rows"]) == 2
from dateutil import parser
timestamps = [parser.parse(row[0]).replace(tzinfo=None) for row in data["row
s"]]
assert timestamps == [
parser.parse("2024-01-02 00:00:00"),
parser.parse("2024-01-02 03:00:00"),
]
def test_order_by(page: Any, server_url: str) -> None:
data = run_query(
page,
server_url,
start="2024-01-01 00:00:00",
end="2024-01-03 00:00:00",
order_by="value",
order_dir="DESC",
limit=100,
)
values = [row[2] for row in data["rows"]]
assert values == sorted(values, reverse=True)
def test_limit(page: Any, server_url: str) -> None:
data = run_query(
page,
server_url,
start="2024-01-01 00:00:00",
end="2024-01-03 00:00:00",
order_by="timestamp",
limit=2,
)
assert len(data["rows"]) == 2
def test_time_column_dropdown(page: Any, server_url: str) -> None:
page.goto(server_url)options = page.locator("#time_column option").all_inner_texts()
assert "timestamp" in options
assert "value" in options
assert page.input_value("#time_column") == "timestamp"
def test_simple_filter(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
page.click("text=Add Filter")
filter_el = page.query_selector("#filters .filter:last-child")
assert filter_el
page.evaluate(
"arg => setSelectValue(arg.el.querySelector('.f-col'), arg.val)",
{"el": filter_el, "val": "user"},
)
val_input = filter_el.query_selector(".f-val")
val_input.click()
page.keyboard.type("alice")
page.keyboard.press("Enter")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
data = page.evaluate("window.lastResults")
assert len(data["rows"]) == 2
assert all(row[3] == "alice" for row in data["rows"])
def test_default_filter_and_layout(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
count = page.evaluate("document.querySelectorAll('#filters .filter').length"
)
assert count == 1
last_is_button = page.evaluate(
"document.querySelector('#filters').lastElementChild.id === 'add_filter'
"
)
assert last_is_button
position = page.evaluate(
"getComputedStyle(document.querySelector('#filters .filter button.remove
')).position"
)
bash-5.2#I’ll add test_time_unit_dropdown similar to test_time_column_dropdown to check options.
[project]
name = "scubaduck"
version = "0.1.0"
description = "Scuba on DuckDB"
readme = "README.md"
requires-python = ">=3.12"
license = {text = "MIT"}
dependencies = [
"duckdb",
"flask",
"python-dateutil>=2.9.0.post0",
]
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"
[tool.pytest.ini_options]
addopts = "-n auto --tb=native"
[tool.pyright]
# Pyright configuration with strict settings
include = ["scubaduck", "tests"]
exclude = ["**/__pycache__", "dist"]
stubPath = "stubs"
venvPath = "."
venv = ".venv"
reportMissingImports = true
reportMissingTypeStubs = true
pythonVersion = "3.12"
pythonPlatform = "All"
typeCheckingMode = "strict"
reportUnknownMemberType = true
reportUnknownParameterType = true
reportUnknownVariableType = true
reportUnknownArgumentType = true
reportPrivateImportUsage = true
reportUntypedFunctionDecorator = true
reportFunctionMemberAccess = true
reportIncompatibleMethodOverride = true
[tool.ruff]
target-version = "py312"
[tool.uv]
dev-dependencies = [
"pytest>=8.3.5",
"ruff>=0.11.2",
"pytest-xdist>=3.6.1",
"playwright>=1.52.0",
"pyright>=1.1.350",
"pytest-playwright>=0.7.0",
]
bash-5.2#I'll add a new property to the QueryParams dataclass and modify the necessary queries.
bash-5.2# git status --short
bash-5.2#Time to activate the virtual environment and get everything set up.
bash-5.2# source .venv/bin/activate
(scubaduck) bash-5.2#Time to add QueryParams.time_unit to the server.py file. Let's do it!
(scubaduck) bash-5.2# grep -n "time_unit" -n scubaduck/server.py
(scubaduck) bash-5.2#I’ll go ahead and patch this through using apply_patch.
(scubaduck) bash-5.2# sed -n '10,40p' scubaduck/server.py
from pathlib import Path
import os
import sqlite3
import traceback
import math
import duckdb
from dateutil import parser as dtparser
from flask import Flask, jsonify, request, send_from_directory
@dataclass
class Filter:
column: str
op: str
value: str | int | float | list[str] | None
@dataclass
class QueryParams:
start: str | None = None
end: str | None = None
order_by: str | None = None
order_dir: str = "ASC"
limit: int | None = None
columns: list[str] = field(default_factory=lambda: [])
filters: list[Filter] = field(default_factory=lambda: [])
derived_columns: dict[str, str] = field(default_factory=lambda: {})
graph_type: str = "samples"
group_by: list[str] = field(default_factory=lambda: [])
aggregate: str | None = Nonex_axis: str | None = None
granularity: str = "Auto"
fill: str = "0"
table: str = "events"
time_column: str = "timestamp"
def _normalize_sqlite_type(sql: str) -> str:
"""Map arbitrary SQLite column types to DuckDB-compatible types."""
t = sql.strip().upper()
if "(" in t:
t = t.split("(", 1)[0]
if "INT" in t:
# SQLite only has a single INTEGER type which is always 64-bit.
# Use DuckDB's BIGINT to avoid overflow when values exceed INT32.
return "BIGINT"
if any(key in t for key in ("CHAR", "CLOB", "TEXT")):
return "VARCHAR"
if "BLOB" in t:
return "BLOB"
if any(key in t for key in ("DOUBLE", "REAL", "FLOA", "NUMERIC", "DECIMAL"))
:
return "DOUBLE"
if "BOOL" in t:
return "BOOLEAN"
if "DATE" in t or "TIME" in t:
return "TIMESTAMP" if "TIME" in t else "DATE"
return "VARCHAR"
def _load_database(path: Path) -> duckdb.DuckDBPyConnection:
if not path.exists():
raise FileNotFoundError(path)
ext = path.suffix.lower()
if ext == ".csv":
con = duckdb.connect()
con.execute(
f"CREATE TABLE events AS SELECT * FROM read_csv_auto('{path.as_posix
()}')"
)
(scubaduck) bash-5.2#I'll add time_unit after time_column for correct field order and parsing.
> *** Begin Patch
> *** Update File: scubaduck/server.py
> @@
> fill: str = "0"
> table: str = "events"
> - time_column: str = "timestamp"
> + time_column: str = "timestamp"
> + time_unit: str = "s"
> *** End Patch
> PATCH
Success. Updated the following files:
M scubaduck/server.py
(scubaduck) bash-5.2#Time to update _time_expr with the latest changes. Let's get this done!
(scubaduck) bash-5.2# sed -n '180,215p' scubaduck/server.py
if gran in mapping:
return mapping[gran]
if gran in {"auto", "fine"} and start and end:
try:
s = dtparser.parse(start)
e = dtparser.parse(end)
except Exception:
return 3600
total = max((e - s).total_seconds(), 1)
buckets = 100 if gran == "auto" else 500
return max(int(total // buckets), 1)
return 3600
def _time_expr(col: str, column_types: Dict[str, str] | None) -> str:
"""Return SQL expression for column interpreted as timestamp."""
if column_types is None:
return col
ctype = column_types.get(col, "").upper()
if not any(t in ctype for t in ["TIMESTAMP", "DATE", "TIME"]):
if any(
t in ctype
for t in [
"INT",
"DECIMAL",
"REAL",
"DOUBLE",
"FLOAT",
"NUMERIC",
"HUGEINT",
]
):
return f"TIMESTAMP 'epoch' + INTERVAL '1 second' * {col}"
return col
(scubaduck) bash-5.2#I’ll adjust the code to accept units and patch the changes.
> *** Update File: scubaduck/server.py
> @@
> -def _time_expr(col: str, column_types: Dict[str, str] | None) -> str:
> - """Return SQL expression for column interpreted as timestamp."""
> +def _time_expr(col: str, column_types: Dict[str, str] | None, unit: str) -> s
tr:
> + """Return SQL expression for column interpreted as timestamp."""
> if column_types is None:
> return col
> ctype = column_types.get(col, "").upper()
> if not any(t in ctype for t in ["TIMESTAMP", "DATE", "TIME"]):
> if any(
> @@
> - ):
> - return f"TIMESTAMP 'epoch' + INTERVAL '1 second' * {col}"
> + ):
> + mult = {
> + "s": "1",
> + "ms": "0.001",
> + "us": "0.000001",
> + "ns": "0.000000001",
> + }.get(unit, "1")
> + return f"TIMESTAMP 'epoch' + INTERVAL '1 second' * ({col} * {mult
})"
> return col
> *** End Patch
> PATCH
Success. Updated the following files:
M scubaduck/server.py
(scubaduck) bash-5.2#I need to search through the code and update instances of _time_expr.
(scubaduck) bash-5.2# grep -n "_time_expr" -n scubaduck/server.py
194:def _time_expr(col: str, column_types: Dict[str, str] | None, unit: str) ->
str:
228: xexpr = _time_expr(x_axis, column_types)
304: time_expr = _time_expr(params.time_column, column_types)
(scubaduck) bash-5.2#I updated the _time_expr signature, so it's time to adjust the line numbers.
(scubaduck) bash-5.2# sed -n '220,240p' scubaduck/server.py
def build_query(params: QueryParams, column_types: Dict[str, str] | None = None)
-> str:
select_parts: list[str] = []
group_cols = params.group_by[:]
if params.graph_type == "timeseries":
sec = _granularity_seconds(params.granularity, params.start, params.end)
x_axis = params.x_axis or params.time_column
xexpr = _time_expr(x_axis, column_types)
if params.start:
bucket_expr = (
f"TIMESTAMP '{params.start}' + INTERVAL '{sec} second' * "
f"CAST(floor((epoch({xexpr}) - epoch(TIMESTAMP '{params.start}')
)/{sec}) AS BIGINT)"
)
else:
bucket_expr = (
f"TIMESTAMP 'epoch' + INTERVAL '{sec} second' * "
f"CAST(floor(epoch({xexpr})/{sec}) AS BIGINT)"
)
select_parts.append(f"{bucket_expr} AS bucket")
group_cols = ["bucket"] + group_cols
(scubaduck) bash-5.2#I'll update the _time_expr call to include params.time_unit for both cases.
xexpr = _time_expr(x_axis, column_types)
if params.start:
bucket_expr = (
f"TIMESTAMP '{params.start}' + INTERVAL '{sec} second' * "
f"CAST(floor((epoch({xexpr}) - epoch(TIMESTAMP '{params.start}')
)/{sec}) AS BIGINT)"
)
else:
bucket_expr = (
f"TIMESTAMP 'epoch' + INTERVAL '{sec} second' * "
f"CAST(floor(epoch({xexpr})/{sec}) AS BIGINT)"
)
select_parts.append(f"{bucket_expr} AS bucket")
group_cols = ["bucket"] + group_cols
has_agg = bool(group_cols) or params.aggregate is not None
if has_agg:
select_cols = (
group_cols[1:] if params.graph_type == "timeseries" else group_cols(scubaduck) bash-5.2# apply_patch<<'PATCH'
> *** Begin Patch
> *** Update File: scubaduck/server.py
> @@
> - xexpr = _time_expr(x_axis, column_types)
> + xexpr = _time_expr(x_axis, column_types, params.time_unit)
> *** End Patch
> PATCH
Success. Updated the following files:
M scubaduck/server.py(scubaduck) bash-5.2# grep -n "_time_expr(" -n scubaduck/server.py
194:def _time_expr(col: str, column_types: Dict[str, str] | None, unit: str) ->
str:
228: xexpr = _time_expr(x_axis, column_types, params.time_unit)
304: time_expr = _time_expr(params.time_column, column_types)
(scubaduck) bash-5.2#I'll make changes around line 304 of the code.
(scubaduck) bash-5.2# sed -n '296,306p' scubaduck/server.py
lines.append(f"LIMIT {params.limit}")
return "\n".join(lines)
for name, expr in params.derived_columns.items():
select_parts.append(f"{expr} AS {name}")
select_clause = ", ".join(select_parts) if select_parts else "*"
lines = [f"SELECT {select_clause}", f'FROM "{params.table}"']
where_parts: list[str] = []
time_expr = _time_expr(params.time_column, column_types)
if params.start:
where_parts.append(f"{time_expr} >= '{params.start}'")(scubaduck) bash-5.2# apply_patch<<'PATCH'
> *** Begin Patch
> *** Update File: scubaduck/server.py
> @@
> - time_expr = _time_expr(params.time_column, column_types)
> + time_expr = _time_expr(params.time_column, column_types, params.time_unit
)
> *** End Patch
> PATCH
Success. Updated the following files:
M scubaduck/server.py
(scubaduck) bash-5.2#I've updated _time_expr and QueryParams; now I need to adjust the create_app route.
(scubaduck) bash-5.2# sed -n '432,470p' scubaduck/server.py
_cache_set(key, values)
return jsonify(values)
@app.route("/api/query", methods=["POST"])
def query() -> Any: # pyright: ignore[reportUnusedFunction]
payload = request.get_json(force=True)
try:
start = parse_time(payload.get("start"))
end = parse_time(payload.get("end"))
except Exception as exc:
return jsonify({"error": str(exc)}), 400
params = QueryParams(
start=start,
end=end,
order_by=payload.get("order_by"),
order_dir=payload.get("order_dir", "ASC"),
limit=payload.get("limit"),
columns=payload.get("columns", []),
derived_columns=payload.get("derived_columns", {}),
graph_type=payload.get("graph_type", "samples"),
group_by=payload.get("group_by", []),
aggregate=payload.get("aggregate"),
show_hits=payload.get("show_hits", False),
x_axis=payload.get("x_axis"),
granularity=payload.get("granularity", "Auto"),
fill=payload.get("fill", "0"),
table=payload.get("table", default_table),
time_column=payload.get("time_column", "timestamp"),
)
for f in payload.get("filters", []):
params.filters.append(Filter(f["column"], f["op"], f.get("value")))
if params.table not in tables:
return jsonify({"error": "Invalid table"}), 400
column_types = get_columns(params.table)
if params.time_column not in column_types:
(scubaduck) bash-5.2#I'll add time_unit after time_column and include validation. Time to patch this!
(scubaduck) bash-5.2# apply_patch<<'PATCH'
> *** Begin Patch
> *** Update File: scubaduck/server.py
> @@
> fill=payload.get("fill", "0"),
> table=payload.get("table", default_table),
> time_column=payload.get("time_column", "timestamp"),
> + time_unit=payload.get("time_unit", "s"),
> )
> *** End Patch
> PATCH
Success. Updated the following files:
M scubaduck/server.py
(scubaduck) bash-5.2#I'll check for valid time_unit after retrieving column types, specifically after validating time_column.
(scubaduck) bash-5.2# sed -n '468,490p' scubaduck/server.py
column_types = get_columns(params.table)
if params.time_column not in column_types:
return jsonify({"error": "Invalid time_column"}), 400
if params.graph_type not in {"table", "timeseries"} and (
params.group_by or params.aggregate or params.show_hits
):
return (
jsonify(
{
"error": "group_by, aggregate and show_hits are only val
id for table or timeseries view"
}
),
400,
)
valid_cols = set(column_types.keys())
valid_cols.update(params.derived_columns.keys())
if params.graph_type == "timeseries":
if params.x_axis is None:
params.x_axis = params.time_column
(scubaduck) bash-5.2#I should add validation after I confirm the time_column looks correct.
(scubaduck) bash-5.2# apply_patch<<'PATCH'
> *** Begin Patch
> *** Update File: scubaduck/server.py
> @@
> column_types = get_columns(params.table)
>
> if params.time_column not in column_types:
> return jsonify({"error": "Invalid time_column"}), 400
> +
> + if params.time_unit not in {"s", "ms", "us", "ns"}:
> + return jsonify({"error": "Invalid time_unit"}), 400
> *** End Patch
> PATCH
Success. Updated the following files:
M scubaduck/server.py
(scubaduck) bash-5.2#I need to modify the server's QueryParams dataclass and update the frontend with time_unit.
(scubaduck) bash-5.2# sed -n '188,208p' scubaduck/static/index.html
<button class="tab active" data-tab="settings">View Settings</button>
<button class="tab" data-tab="columns">Columns</button>
<button id="dive" onclick="dive()">Dive</button>
</div>
<div id="settings" class="tab-content active">
<div class="field">
<label>Time Column</label>
<select id="time_column"></select>
</div>
<div class="field">
<label>Start<span class="help" title="Sets the start/end of the time r
ange to query. Can be any kind of datetime string. For example: 'April 23, 2014'
or 'yesterday'.">[?]</span></label>
<div class="rel-box">
<input id="start" type="text" />
<button type="button" class="rel-btn" data-target="start-select">	
660;</button>
<div id="start-select" class="rel-dropdown" data-input="start">
<div data-value="-1 hour">-1 hour</div>
<div data-value="-3 hours">-3 hours</div>
<div data-value="-12 hours">-12 hours</div>
<div data-value="-1 day">-1 day</div>
<div data-value="-3 days">-3 days</div>
<div data-value="-1 week">-1 week</div>
(scubaduck) bash-5.2#I'll add the new select statement right after the time_column.
(scubaduck) bash-5.2# apply_patch<<'PATCH'
> *** Begin Patch
> *** Update File: scubaduck/static/index.html
> @@
> - <div class="field">
> - <label>Time Column</label>
> - <select id="time_column"></select>
> - </div>
> + <div class="field">
> + <label>Time Column</label>
> + <select id="time_column"></select>
> + <select id="time_unit" style="margin-left:4px">
> + <option value="s">s</option>
> + <option value="ms">ms</option>
> + <option value="us">us</option>
> + <option value="ns">ns</option>
> + </select>
> + </div>
> *** End Patch
> PATCH
Success. Updated the following files:
M scubaduck/static/index.html
(scubaduck) bash-5.2#Time to patch the JS functions, starting with 'collectParams' and others.
(scubaduck) bash-5.2# grep -n "collectParams" -n scubaduck/static/index.html
931: const params = collectParams();
960:function collectParams() {(scubaduck) bash-5.2# sed -n '960,995p' scubaduck/static/index.html
function collectParams() {
updateSelectedColumns();
const payload = {
table: document.getElementById('table').value,
time_column: document.getElementById('time_column').value,
start: document.getElementById('start').value,
end: document.getElementById('end').value,
order_by: document.getElementById('order_by').value,
order_dir: orderDir,
limit: parseInt(document.getElementById('limit').value, 10),
columns: selectedColumns.filter(c =>
c !== 'Hits' && !derivedColumns.some(dc => dc.name === c)
),
graph_type: graphTypeSel.value,
filters: Array.from(document.querySelectorAll('#filters .filter')).map(f =>
{
const chips = f.chips || [];
const op = f.querySelector('.f-op').value;
let value = null;
if (op !== 'empty' && op !== '!empty') {
value = chips.length === 0 ? null : (chips.length === 1 ? chips[0] : chi
ps);
}
const colSel = f.querySelector('.f-col select') || f.querySelector('.f-col
');
return {column: colSel.value, op, value};
}),
derived_columns: Array.from(document.querySelectorAll('#derived_list .derive
d')).map(d => ({
type: d.querySelector('.d-type').value,
name: d.querySelector('.d-name').value,
expr: d.querySelector('.d-expr').value,
include: d.querySelector('.d-use').checked,
}))
};
if (graphTypeSel.value === 'table' || graphTypeSel.value === 'timeseries') {
payload.group_by = groupBy.chips || [];
payload.aggregate = document.getElementById('aggregate').value;
payload.show_hits = document.getElementById('show_hits').checked;
}(scubaduck) bash-5.2# sed -n '995,1034p' scubaduck/static/index.html
}
if (graphTypeSel.value === 'timeseries') {
payload.x_axis = document.getElementById('x_axis').value;
payload.granularity = document.getElementById('granularity').value;
payload.fill = document.getElementById('fill').value;
}
return payload;
}
function paramsToSearch(params) {
const sp = new URLSearchParams();
if (params.table) sp.set('table', params.table);
if (params.time_column) sp.set('time_column', params.time_column);
if (params.start) sp.set('start', params.start);
if (params.end) sp.set('end', params.end);
if (params.order_by) sp.set('order_by', params.order_by);
if (params.order_dir) sp.set('order_dir', params.order_dir);
if (params.limit !== null && params.limit !== undefined) sp.set('limit', param
s.limit);
if (params.columns && params.columns.length) sp.set('columns', params.columns.
join(','));
if (params.filters && params.filters.length) sp.set('filters', JSON.stringify(
params.filters));
if (params.derived_columns && params.derived_columns.length) sp.set('derived_c
olumns', JSON.stringify(params.derived_columns));
if (params.graph_type) sp.set('graph_type', params.graph_type);
if (params.graph_type === 'table' || params.graph_type === 'timeseries') {
if (params.group_by && params.group_by.length) sp.set('group_by', params.gro
up_by.join(','));
if (params.aggregate) sp.set('aggregate', params.aggregate);
if (params.show_hits) sp.set('show_hits', '1');
}
if (params.graph_type === 'timeseries') {
if (params.x_axis) sp.set('x_axis', params.x_axis);
if (params.granularity) sp.set('granularity', params.granularity);
if (params.fill) sp.set('fill', params.fill);
}
const qs = sp.toString();
return qs ? '?' + qs : '';
}
function applyParams(params) {
if (params.table) document.getElementById('table').value = params.table;
document.getElementById('time_column').value = params.time_column || defaultTi
meColumn;
document.getElementById('start').value = params.start || '';(scubaduck) bash-5.2# sed -n '1034,1076p' scubaduck/static/index.html
document.getElementById('start').value = params.start || '';
document.getElementById('end').value = params.end || '';
if (params.order_by) {
document.getElementById('order_by').value = params.order_by;
}
orderDir = params.order_dir || 'ASC';
updateOrderDirButton();
if (params.limit !== undefined && params.limit !== null) {
document.getElementById('limit').value = params.limit;
}
graphTypeSel.value = params.graph_type || 'samples';
updateDisplayTypeUI();
if (params.x_axis) document.getElementById('x_axis').value = params.x_axis;
if (params.granularity) document.getElementById('granularity').value = params.
granularity;
if (params.fill) document.getElementById('fill').value = params.fill;
if (params.group_by) {
groupBy.chips.splice(0, groupBy.chips.length, ...params.group_by);
groupBy.renderChips();
}
if (params.aggregate) document.getElementById('aggregate').value = params.aggr
egate;
document.getElementById('show_hits').checked = params.show_hits ?? true;
document.querySelectorAll('#column_groups input').forEach(cb => {
cb.checked = !params.columns || params.columns.includes(cb.value);
});
updateSelectedColumns();
const dlist = document.getElementById('derived_list');
dlist.innerHTML = '';
derivedColumns.splice(0, derivedColumns.length);
if (params.derived_columns && params.derived_columns.length) {
params.derived_columns.forEach(dc => addDerived(dc));
}
refreshDerivedColumns();
const list = document.getElementById('filter_list');
list.innerHTML = '';
if (params.filters && params.filters.length) {
params.filters.forEach(f => {
addFilter();
const el = list.lastElementChild;
const colSel = el.querySelector('.f-col select') || el.querySelector('.f-c
ol');
colSel.value = f.column;
colSel.dispatchEvent(new Event('change'));
el.querySelector('.f-op').value = f.op;
el.querySelector('.f-op').dispatchEvent(new Event('change'));
(scubaduck) bash-5.2#I'm continuing to scroll until I find parseSearch.
(scubaduck) bash-5.2# sed -n '1076,1124p' scubaduck/static/index.html
el.querySelector('.f-op').dispatchEvent(new Event('change'));
if (f.value !== null && f.op !== 'empty' && f.op !== '!empty') {
const values = Array.isArray(f.value) ? f.value : [f.value];
values.forEach(v => el.addChip(v));
el.renderChips();
}
});
} else {
addFilter();
}
}
function parseSearch() {
const sp = new URLSearchParams(window.location.search);
const params = {};
if (sp.has('table')) params.table = sp.get('table');
if (sp.has('time_column')) params.time_column = sp.get('time_column');
if (sp.has('start')) params.start = sp.get('start');
if (sp.has('end')) params.end = sp.get('end');
if (sp.has('order_by')) params.order_by = sp.get('order_by');
if (sp.has('order_dir')) params.order_dir = sp.get('order_dir');
if (sp.has('limit')) params.limit = parseInt(sp.get('limit'), 10);
if (sp.has('columns')) params.columns = sp.get('columns').split(',').filter(c
=> c);
if (sp.has('filters')) {
try { params.filters = JSON.parse(sp.get('filters')); } catch(e) { params.fi
lters = []; }
}
if (sp.has('graph_type')) params.graph_type = sp.get('graph_type');
if (sp.has('group_by')) params.group_by = sp.get('group_by').split(',').filter
(c => c);
if (sp.has('aggregate')) params.aggregate = sp.get('aggregate');
if (sp.has('show_hits')) params.show_hits = sp.get('show_hits') === '1';
if (sp.has('x_axis')) params.x_axis = sp.get('x_axis');
if (sp.has('granularity')) params.granularity = sp.get('granularity');
if (sp.has('fill')) params.fill = sp.get('fill');
if (sp.has('derived_columns')) {
try { params.derived_columns = JSON.parse(sp.get('derived_columns')); } catc
h(e) { params.derived_columns = []; }
}
return params;
}
function initFromUrl() {
const params = parseSearch();
history.replaceState(params, '', paramsToSearch(params));
applyParams(params);
dive(false);
}
window.addEventListener('popstate', e => {
const params = e.state || parseSearch();
applyParams(params);(scubaduck) bash-5.2# sed -n '1124,1170p' scubaduck/static/index.html
applyParams(params);
dive(false);
});
let originalRows = [];
let sortState = {index: null, dir: null};
function renderTable(rows) {
const table = document.getElementById('results');
table.innerHTML = '';
if (rows.length === 0) return;
let hitsIndex = selectedColumns.indexOf('Hits');
let totalHits = 0;
if (hitsIndex !== -1) {
totalHits = rows.reduce((s, r) => s + Number(r[hitsIndex]), 0);
}
const header = document.createElement('tr');
selectedColumns.forEach((col, i) => {
const th = document.createElement('th');
let label = col;
if (
displayType === 'table' &&
col !== 'Hits' &&
!(groupBy.chips || []).includes(col)
) {
const agg = document.getElementById('aggregate').value.toLowerCase();
label += ` (${agg})`;
}
th.textContent = label;
th.dataset.index = i;
th.addEventListener('click', handleSort);
if (sortState.index === i) {
th.classList.add('sorted');
th.textContent =
label + (sortState.dir === 'desc' ? ' \u25BC' : ' \u25B2');
}
if (!isStringColumn(col)) th.style.textAlign = 'right';
header.appendChild(th);
});
table.appendChild(header);
rows.forEach(row => {
const tr = document.createElement('tr');
tr.addEventListener('click', () => {
const wasSelected = tr.classList.contains('selected');
document
.querySelectorAll('#results tr.selected')
.forEach(el => el.classList.remove('selected'));(scubaduck) bash-5.2# grep -n "new Date(Number" -n scubaduck/static/index.html
1184: d = new Date(Number(v) * 1000);(scubaduck) bash-5.2# sed -n '1178,1192p' scubaduck/static/index.html
if (isTimeColumn(col)) {
let d;
const t = (columnTypes[col] || '').toUpperCase();
if (t.includes('TIMESTAMP') || t.includes('DATE') || t.includes('TIME'))
{
d = new Date(v);
} else {
d = new Date(Number(v) * 1000);
}
td.textContent = d.toLocaleString('en-US', {
weekday: 'short',
month: 'short',
day: 'numeric',
year: 'numeric',
hour: 'numeric',
minute: 'numeric',(scubaduck) bash-5.2# apply_patch<<'PATCH'
> *** Begin Patch
> *** Update File: scubaduck/static/index.html
> @@
> - if (t.includes('TIMESTAMP') || t.includes('DATE') || t.includes('TIME
')) {
> - d = new Date(v);
> - } else {
> - d = new Date(Number(v) * 1000);
> - }
> + if (t.includes('TIMESTAMP') || t.includes('DATE') || t.includes('TIME
')) {
> + d = new Date(v);
> + } else {
> + const unit = document.getElementById('time_unit').value;
> + const factors = {s: 1000, ms: 1, us: 0.001, ns: 0.000001};
> + d = new Date(Number(v) * (factors[unit] || 1000));
> + }
> *** End Patch
> PATCH
Success. Updated the following files:
M scubaduck/static/index.html
(scubaduck) bash-5.2#I'll update parseSearch, applyParams, and others to handle the time_unit in collectParams.
(scubaduck) bash-5.2# grep -n "time_unit" -n scubaduck/static/index.html
196: <select id="time_unit" style="margin-left:4px">
1184: const unit = document.getElementById('time_unit').value;
(scubaduck) bash-5.2#I’ll expand collectParams since it only includes time_unit right now. Let’s enhance it!
(scubaduck) bash-5.2# sed -n '964,1000p' scubaduck/static/index.html
time_column: document.getElementById('time_column').value,
start: document.getElementById('start').value,
end: document.getElementById('end').value,
order_by: document.getElementById('order_by').value,
order_dir: orderDir,
limit: parseInt(document.getElementById('limit').value, 10),
columns: selectedColumns.filter(c =>
c !== 'Hits' && !derivedColumns.some(dc => dc.name === c)
),
graph_type: graphTypeSel.value,
filters: Array.from(document.querySelectorAll('#filters .filter')).map(f =>
{
const chips = f.chips || [];
const op = f.querySelector('.f-op').value;
let value = null;
if (op !== 'empty' && op !== '!empty') {
value = chips.length === 0 ? null : (chips.length === 1 ? chips[0] : chi
ps);
}
const colSel = f.querySelector('.f-col select') || f.querySelector('.f-col
');
return {column: colSel.value, op, value};
}),
derived_columns: Array.from(document.querySelectorAll('#derived_list .derive
d')).map(d => ({
type: d.querySelector('.d-type').value,
name: d.querySelector('.d-name').value,
expr: d.querySelector('.d-expr').value,
include: d.querySelector('.d-use').checked,
}))
};
if (graphTypeSel.value === 'table' || graphTypeSel.value === 'timeseries') {
payload.group_by = groupBy.chips || [];
payload.aggregate = document.getElementById('aggregate').value;
payload.show_hits = document.getElementById('show_hits').checked;
}
if (graphTypeSel.value === 'timeseries') {
payload.x_axis = document.getElementById('x_axis').value;
payload.granularity = document.getElementById('granularity').value;
payload.fill = document.getElementById('fill').value;
}
(scubaduck) bash-5.2#I'm adding time_unit: document.getElementById('time_unit').value, after time_column.
(scubaduck) bash-5.2# apply_patch<<'PATCH'
> *** Begin Patch
> *** Update File: scubaduck/static/index.html
> @@
> - time_column: document.getElementById('time_column').value,
> + time_column: document.getElementById('time_column').value,
> + time_unit: document.getElementById('time_unit').value,
> *** End Patch
> PATCH
Success. Updated the following files:
M scubaduck/static/index.html(scubaduck) bash-5.2# grep -n "paramsToSearch" -n scubaduck/static/index.html
933: history.pushState(params, '', paramsToSearch(params));
1005:function paramsToSearch(params) {
1118: history.replaceState(params, '', paramsToSearch(params));(scubaduck) bash-5.2# sed -n '1005,1034p' scubaduck/static/index.html
function paramsToSearch(params) {
const sp = new URLSearchParams();
if (params.table) sp.set('table', params.table);
if (params.time_column) sp.set('time_column', params.time_column);
if (params.start) sp.set('start', params.start);
if (params.end) sp.set('end', params.end);
if (params.order_by) sp.set('order_by', params.order_by);
if (params.order_dir) sp.set('order_dir', params.order_dir);
if (params.limit !== null && params.limit !== undefined) sp.set('limit', param
s.limit);
if (params.columns && params.columns.length) sp.set('columns', params.columns.
join(','));
if (params.filters && params.filters.length) sp.set('filters', JSON.stringify(
params.filters));
if (params.derived_columns && params.derived_columns.length) sp.set('derived_c
olumns', JSON.stringify(params.derived_columns));
if (params.graph_type) sp.set('graph_type', params.graph_type);
if (params.graph_type === 'table' || params.graph_type === 'timeseries') {
if (params.group_by && params.group_by.length) sp.set('group_by', params.gro
up_by.join(','));
if (params.aggregate) sp.set('aggregate', params.aggregate);
if (params.show_hits) sp.set('show_hits', '1');
}
if (params.graph_type === 'timeseries') {
if (params.x_axis) sp.set('x_axis', params.x_axis);
if (params.granularity) sp.set('granularity', params.granularity);
if (params.fill) sp.set('fill', params.fill);
}
const qs = sp.toString();
return qs ? '?' + qs : '';
}
function applyParams(params) {
if (params.table) document.getElementById('table').value = params.table;
document.getElementById('time_column').value = params.time_column || defaultTi
meColumn;(scubaduck) bash-5.2# apply_patch<<'PATCH'
> *** Begin Patch
> *** Update File: scubaduck/static/index.html
> @@
> if (params.time_column) sp.set('time_column', params.time_column);
> + if (params.time_unit) sp.set('time_unit', params.time_unit);
> *** End Patch
> PATCH
Success. Updated the following files:
M scubaduck/static/index.html
(scubaduck) bash-5.2#I'll move on to implementing the parseSearch function now.
(scubaduck) bash-5.2# grep -n "time_unit" -n scubaduck/static/index.html
196: <select id="time_unit" style="margin-left:4px">
965: time_unit: document.getElementById('time_unit').value,
1009: if (params.time_unit) sp.set('time_unit', params.time_unit);
1186: const unit = document.getElementById('time_unit').value;
(scubaduck) bash-5.2#I added the patch at 964 earlier. Let’s find and update parseSearch.
(scubaduck) bash-5.2# sed -n '1096,1128p' scubaduck/static/index.html
if (sp.has('end')) params.end = sp.get('end');
if (sp.has('order_by')) params.order_by = sp.get('order_by');
if (sp.has('order_dir')) params.order_dir = sp.get('order_dir');
if (sp.has('limit')) params.limit = parseInt(sp.get('limit'), 10);
if (sp.has('columns')) params.columns = sp.get('columns').split(',').filter(c
=> c);
if (sp.has('filters')) {
try { params.filters = JSON.parse(sp.get('filters')); } catch(e) { params.fi
lters = []; }
}
if (sp.has('graph_type')) params.graph_type = sp.get('graph_type');
if (sp.has('group_by')) params.group_by = sp.get('group_by').split(',').filter
(c => c);
if (sp.has('aggregate')) params.aggregate = sp.get('aggregate');
if (sp.has('show_hits')) params.show_hits = sp.get('show_hits') === '1';
if (sp.has('x_axis')) params.x_axis = sp.get('x_axis');
if (sp.has('granularity')) params.granularity = sp.get('granularity');
if (sp.has('fill')) params.fill = sp.get('fill');
if (sp.has('derived_columns')) {
try { params.derived_columns = JSON.parse(sp.get('derived_columns')); } catc
h(e) { params.derived_columns = []; }
}
return params;
}
function initFromUrl() {
const params = parseSearch();
history.replaceState(params, '', paramsToSearch(params));
applyParams(params);
dive(false);
}
window.addEventListener('popstate', e => {
const params = e.state || parseSearch();
applyParams(params);
dive(false);
});
(scubaduck) bash-5.2#I'll include the retrieval of time_unit in the process now.
(scubaduck) bash-5.2# apply_patch<<'PATCH'
> *** Begin Patch
> *** Update File: scubaduck/static/index.html
> @@
> if (sp.has('time_column')) params.time_column = sp.get('time_column');
> + if (sp.has('time_unit')) params.time_unit = sp.get('time_unit');
> *** End Patch
> PATCH
Success. Updated the following files:
M scubaduck/static/index.html
(scubaduck) bash-5.2#It's time to update applyParams. Let's make the necessary changes.
(scubaduck) bash-5.2# grep -n "applyParams" -n scubaduck/static/index.html
681: applyParams(parseSearch());
1033:function applyParams(params) {
1121: applyParams(params);
1127: applyParams(params);(scubaduck) bash-5.2# sed -n '1032,1060p' scubaduck/static/index.html
function applyParams(params) {
if (params.table) document.getElementById('table').value = params.table;
document.getElementById('time_column').value = params.time_column || defaultTi
meColumn;
document.getElementById('start').value = params.start || '';
document.getElementById('end').value = params.end || '';
if (params.order_by) {
document.getElementById('order_by').value = params.order_by;
}
orderDir = params.order_dir || 'ASC';
updateOrderDirButton();
if (params.limit !== undefined && params.limit !== null) {
document.getElementById('limit').value = params.limit;
}
graphTypeSel.value = params.graph_type || 'samples';
updateDisplayTypeUI();
if (params.x_axis) document.getElementById('x_axis').value = params.x_axis;
if (params.granularity) document.getElementById('granularity').value = params.
granularity;
if (params.fill) document.getElementById('fill').value = params.fill;
if (params.group_by) {
groupBy.chips.splice(0, groupBy.chips.length, ...params.group_by);
groupBy.renderChips();
}
if (params.aggregate) document.getElementById('aggregate').value = params.aggr
egate;
document.getElementById('show_hits').checked = params.show_hits ?? true;
document.querySelectorAll('#column_groups input').forEach(cb => {
cb.checked = !params.columns || params.columns.includes(cb.value);
});
updateSelectedColumns();(scubaduck) bash-5.2# sed -n '1060,1088p' scubaduck/static/index.html
updateSelectedColumns();
const dlist = document.getElementById('derived_list');
dlist.innerHTML = '';
derivedColumns.splice(0, derivedColumns.length);
if (params.derived_columns && params.derived_columns.length) {
params.derived_columns.forEach(dc => addDerived(dc));
}
refreshDerivedColumns();
const list = document.getElementById('filter_list');
list.innerHTML = '';
if (params.filters && params.filters.length) {
params.filters.forEach(f => {
addFilter();
const el = list.lastElementChild;
const colSel = el.querySelector('.f-col select') || el.querySelector('.f-c
ol');
colSel.value = f.column;
colSel.dispatchEvent(new Event('change'));
el.querySelector('.f-op').value = f.op;
el.querySelector('.f-op').dispatchEvent(new Event('change'));
if (f.value !== null && f.op !== 'empty' && f.op !== '!empty') {
const values = Array.isArray(f.value) ? f.value : [f.value];
values.forEach(v => el.addChip(v));
el.renderChips();
}
});
} else {
addFilter();
}
}
(scubaduck) bash-5.2#I'll define the time_unit at the beginning, since it's not set anywhere.
(scubaduck) bash-5.2# apply_patch<<'PATCH'
> *** Begin Patch
> *** Update File: scubaduck/static/index.html
> @@
> if (params.table) document.getElementById('table').value = params.table;
> document.getElementById('time_column').value = params.time_column || defaul
tTimeColumn;
> + if (params.time_unit) document.getElementById('time_unit').value = params.t
ime_unit;
> document.getElementById('start').value = params.start || '';
> *** End Patch
> PATCH
Success. Updated the following files:
M scubaduck/static/index.html
(scubaduck) bash-5.2#I'll add a new server test 'test_integer_time_unit_ms' in tests/test_server.py to cover changes.
(scubaduck) bash-5.2# grep -n "integer_time_column" -n tests/test_server.py
319:def test_integer_time_column(tmp_path: Path) -> None:(scubaduck) bash-5.2# sed -n '316,334p' tests/test_server.py
assert data["rows"][0][1] == big_value
def test_integer_time_column(tmp_path: Path) -> None:
csv_file = tmp_path / "events.csv"
csv_file.write_text("created,event\n1704067200,login\n1704070800,logout\n")
app = server.create_app(csv_file)
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-01 01:00:00",
"order_by": "created",
"columns": ["created", "event"],
"time_column": "created",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
))
data = rv.get_json()
assert rv.status_code == 200
assert len(data["rows"]) == 2
def test_envvar_db(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
csv_file = tmp_path / "custom.csv"
csv_file.write_text("timestamp,event,value,user\n2024-01-01 00:00:00,login,5
,bob\n")
monkeypatch.setenv("SCUBADUCK_DB", str(csv_file))
app = server.create_app()
client = app.test_client()
payload = _make_payload()
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
rows = rv.get_json()["rows"]
assert len(rows) == 1
def test_envvar_db_missing(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> N
one:
(scubaduck) bash-5.2#I'll add the new test after the test_integer_time_column. Time for a patch.
> @@
> def test_integer_time_column(tmp_path: Path) -> None:
> @@
> assert rv.status_code == 200
> assert len(data["rows"]) == 2
> +
> +
> +def test_integer_time_unit_ms(tmp_path: Path) -> None:
> + csv_file = tmp_path / "events.csv"
> + csv_file.write_text(
> + "created,event\n1704067200000,login\n1704070800000,logout\n"
> + )
> + app = server.create_app(csv_file)
> + client = app.test_client()
> + payload = {
> + "table": "events",
> + "start": "2024-01-01 00:00:00",
> + "end": "2024-01-01 01:00:00",
> + "order_by": "created",
> + "columns": ["created", "event"],
> + "time_column": "created",
> + "time_unit": "ms",
> + }
> + rv = client.post(
> + "/api/query", data=json.dumps(payload), content_type="application/jso
n"
> + )
> + data = rv.get_json()
> + assert rv.status_code == 200
> + assert len(data["rows"]) == 2
> *** End Patch
> PATCH
Success. Updated the following files:
M tests/test_server.py
(scubaduck) bash-5.2#I’ll modify tests/test_web to include a check for the time_unit dropdown.
(scubaduck) bash-5.2# grep -n "time_column_dropdown" -n tests/test_web.py
99:def test_time_column_dropdown(page: Any, server_url: str) -> None:(scubaduck) bash-5.2# sed -n '92,110p' tests/test_web.py
end="2024-01-03 00:00:00",
order_by="timestamp",
limit=2,
)
assert len(data["rows"]) == 2
def test_time_column_dropdown(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#time_column option", state="attached")
options = page.locator("#time_column option").all_inner_texts()
assert "timestamp" in options
assert "value" in options
assert page.input_value("#time_column") == "timestamp"
def test_simple_filter(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
(scubaduck) bash-5.2#I’ll patch this first, then create a new test for the time unit dropdown.
> *** Begin Patch
> *** Update File: tests/test_web.py
> @@
> def test_time_column_dropdown(page: Any, server_url: str) -> None:
> @@
> assert page.input_value("#time_column") == "timestamp"
> +
> +
> +def test_time_unit_dropdown(page: Any, server_url: str) -> None:
> + page.goto(server_url)
> + page.wait_for_selector("#time_unit", state="attached")
> + opts = page.locator("#time_unit option").all_inner_texts()
> + assert "ms" in opts
> + assert page.input_value("#time_unit") == "s"
> *** End Patch
> PATCH
Success. Updated the following files:
M tests/test_web.py
(scubaduck) bash-5.2#I need to add time_unit to applyParams and parseSearch to finalize the patch.
196: <select id="time_unit" style="margin-left:4px">
965: time_unit: document.getElementById('time_unit').value,
1009: if (params.time_unit) sp.set('time_unit', params.time_unit);
1036: if (params.time_unit) document.getElementById('time_unit').value = params
.time_unit;
1096: if (sp.has('time_unit')) params.time_unit = sp.get('time_unit');
1188: const unit = document.getElementById('time_unit').value;
(scubaduck) bash-5.2#I'll check the lines around 1028-1040 to ensure the patch is correct.
}
const qs = sp.toString();
return qs ? '?' + qs : '';
}
function applyParams(params) {
if (params.table) document.getElementById('table').value = params.table;
document.getElementById('time_column').value = params.time_column || defaultTi
meColumn;
if (params.time_unit) document.getElementById('time_unit').value = params.time
_unit;
document.getElementById('start').value = params.start || '';
document.getElementById('end').value = params.end || '';
if (params.order_by) {
document.getElementById('order_by').value = params.order_by;
(scubaduck) bash-5.2#I'll take a look at the parseSearch patch as well.
(scubaduck) bash-5.2# sed -n '1088,1108p' scubaduck/static/index.html
}
}
function parseSearch() {
const sp = new URLSearchParams(window.location.search);
const params = {};
if (sp.has('table')) params.table = sp.get('table');
if (sp.has('time_column')) params.time_column = sp.get('time_column');
if (sp.has('time_unit')) params.time_unit = sp.get('time_unit');
if (sp.has('start')) params.start = sp.get('start');
if (sp.has('end')) params.end = sp.get('end');
if (sp.has('order_by')) params.order_by = sp.get('order_by');
if (sp.has('order_dir')) params.order_dir = sp.get('order_dir');
if (sp.has('limit')) params.limit = parseInt(sp.get('limit'), 10);
if (sp.has('columns')) params.columns = sp.get('columns').split(',').filter(c
=> c);
if (sp.has('filters')) {
try { params.filters = JSON.parse(sp.get('filters')); } catch(e) { params.fi
lters = []; }
}
if (sp.has('graph_type')) params.graph_type = sp.get('graph_type');
if (sp.has('group_by')) params.group_by = sp.get('group_by').split(',').filter
(c => c);
if (sp.has('aggregate')) params.aggregate = sp.get('aggregate');
(scubaduck) bash-5.2#Checking for a single location of 'time_unit'. Consider updating run_query, though tests don't need it.
(scubaduck) bash-5.2# ruff format scubaduck/server.py scubaduck/static/index.htm
l tests/test_server.py tests/test_web.py >/tmp/format.log && tail -n 20 /tmp/for
mat.log
error: Failed to parse scubaduck/static/index.html:1:1: Expected a statement
(scubaduck) bash-5.2#Ruff can't handle HTML, so I'll only run it on Python files.
(scubaduck) bash-5.2# ruff format scubaduck/server.py tests/test_server.py tests
/test_web.py >/tmp/format.log && tail -n 20 /tmp/format.log
3 files left unchanged(scubaduck) bash-5.2# ruff check scubaduck/server.py scubaduck/static/index.html
tests/test_server.py tests/test_web.py >/tmp/ruff.log && tail -n 20 /tmp/ruff.l
og(scubaduck) bash-5.2# pyright >/tmp/pyright.log && tail -n 20 /tmp/pyright.logTraceback (most recent call last):
File "/usr/lib/python3.12/urllib/request.py", line 1344, in do_open
h.request(req.get_method(), req.selector, req.data, headers,
File "/usr/lib/python3.12/http/client.py", line 1336, in request
self._send_request(method, url, body, headers, encode_chunked)
File "/usr/lib/python3.12/http/client.py", line 1382, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "/usr/lib/python3.12/http/client.py", line 1331, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "/usr/lib/python3.12/http/client.py", line 1091, in _send_output
self.send(msg)
File "/usr/lib/python3.12/http/client.py", line 1035, in send
self.connect()
File "/usr/lib/python3.12/http/client.py", line 1470, in connect
super().connect()
File "/usr/lib/python3.12/http/client.py", line 1001, in connect
self.sock = self._create_connection(
^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3.12/socket.py", line 852, in create_connection
raise exceptions[0]
File "/usr/lib/python3.12/socket.py", line 837, in create_connection
sock.connect(sa)
OSError: [Errno 113] No route to host
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "<frozen runpy>", line 198, in _run_module_as_main
File "<frozen runpy>", line 88, in _run_code
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/nodeenv.py", lin
e 1548, in <module>
main()
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/nodeenv.py", lin
e 1119, in main
args.node = get_last_stable_node_version()
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/nodeenv.py", lin
e 1052, in get_last_stable_node_version
return _get_versions_json()[0]['version'].lstrip('v')
^^^^^^^^^^^^^^^^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/nodeenv.py", lin
e 1028, in _get_versions_json
response = urlopen('%s/index.json' % src_base_url)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/nodeenv.py", lin
e 652, in urlopen
return urllib2.urlopen(req)
^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3.12/urllib/request.py", line 215, in urlopen
return opener.open(url, data, timeout)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3.12/urllib/request.py", line 515, in open
response = self._open(req, data)
^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3.12/urllib/request.py", line 532, in _open
result = self._call_chain(self.handle_open, protocol, protocol +
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3.12/urllib/request.py", line 492, in _call_chain
result = func(*args)
^^^^^^^^^^^
File "/usr/lib/python3.12/urllib/request.py", line 1392, in https_open
return self.do_open(http.client.HTTPSConnection, req,
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3.12/urllib/request.py", line 1347, in do_open
raise URLError(err)
urllib.error.URLError: <urlopen error [Errno 113] No route to host>
Traceback (most recent call last):
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pyright/node.py"
, line 93, in _install_node_env
subprocess.run(args, check=True)
File "/usr/lib/python3.12/subprocess.py", line 571, in run
raise CalledProcessError(retcode, process.args,
subprocess.CalledProcessError: Command '['/workspace/scubaduck/.venv/bin/python'
, '-m', 'nodeenv', '/root/.cache/pyright-python/nodeenv']' returned non-zero exi
t status 1.
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/workspace/scubaduck/.venv/bin/pyright", line 10, in <module>
sys.exit(entrypoint())
^^^^^^^^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pyright/cli.py",
line 31, in entrypoint
sys.exit(main(sys.argv[1:]))
^^^^^^^^^^^^^^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pyright/cli.py",
line 18, in main
return run(*args, **kwargs).returncode
^^^^^^^^^^^^^^^^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pyright/cli.py",
line 27, in run
return node.run('node', str(script), *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pyright/node.py"
, line 138, in run
strategy = _resolve_strategy(target)
^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pyright/node.py"
, line 130, in _resolve_strategy
return NodeenvStrategy(type='nodeenv', path=_ensure_node_env(target))
^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pyright/node.py"
, line 54, in _ensure_node_env
_install_node_env()
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pyright/node.py"
, line 95, in _install_node_env
raise RuntimeError(
RuntimeError: nodeenv failed; for more reliable node.js binaries try `pip instal
l pyright[nodejs]`
(scubaduck) bash-5.2#Pyright failed due to network restrictions. I'll make sure to mention that in the message.
(scubaduck) bash-5.2# pytest -q >/tmp/pytest.log && tail -n 20 /tmp/pytest.logbringing up nodes...
bringing up nodes...
........................F............................................... [ 80%]
................. [100%]
=================================== FAILURES ===================================
__________________________ test_integer_time_unit_ms ___________________________
[gw2] linux -- Python 3.12.3 /workspace/scubaduck/.venv/bin/python
Traceback (most recent call last):
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/runner.p
y", line 341, in from_call
result: TResult | None = func()
^^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/runner.p
y", line 242, in <lambda>
lambda: runtest_hook(item=item, **kwds), when=when, reraise=reraise
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_hooks.py
", line 512, in __call__
return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult
)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_manager.
py", line 120, in _hookexec
return self._inner_hookexec(hook_name, methods, kwargs, firstresult)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_callers.
py", line 167, in _multicall
raise exception
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_callers.
py", line 139, in _multicall
teardown.throw(exception)
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/threadex
ception.py", line 92, in pytest_runtest_call
yield from thread_exception_runtest_hook()
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/threadex
ception.py", line 68, in thread_exception_runtest_hook
yield
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_callers.
py", line 139, in _multicall
teardown.throw(exception)
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/unraisab
leexception.py", line 95, in pytest_runtest_call
yield from unraisable_exception_runtest_hook()
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/unraisab
leexception.py", line 70, in unraisable_exception_runtest_hook
yield
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_callers.
py", line 139, in _multicall
teardown.throw(exception)
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/logging.
py", line 846, in pytest_runtest_call
yield from self._runtest_for(item, "call")
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/logging.
py", line 829, in _runtest_for
yield
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_callers.
py", line 139, in _multicall
teardown.throw(exception)
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/capture.
py", line 898, in pytest_runtest_call
return (yield)
^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_callers.
py", line 139, in _multicall
teardown.throw(exception)
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/skipping
.py", line 257, in pytest_runtest_call
return (yield)
^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_callers.
py", line 121, in _multicall
res = hook_impl.function(*args)
^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/runner.p
y", line 174, in pytest_runtest_call
item.runtest()
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/python.p
y", line 1627, in runtest
self.ihook.pytest_pyfunc_call(pyfuncitem=self)
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_hooks.py
", line 512, in __call__
return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult
)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_manager.
py", line 120, in _hookexec
return self._inner_hookexec(hook_name, methods, kwargs, firstresult)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_callers.
py", line 167, in _multicall
raise exception
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_callers.
py", line 121, in _multicall
res = hook_impl.function(*args)
^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/python.p
y", line 159, in pytest_pyfunc_call
result = testfunction(**testargs)
^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/scubaduck/tests/test_server.py", line 358, in test_integer_ti
me_unit_ms
assert rv.status_code == 200
AssertionError: assert 400 == 200
+ where 400 = <WrapperTestResponse 2209 bytes [400 BAD REQUEST]>.status_code
----------------------------- Captured stdout call -----------------------------
Query failed:
SELECT created, event
FROM "events"
WHERE TIMESTAMP 'epoch' + INTERVAL '1 second' * (created * 0.001) >= '2024-01-01
00:00:00' AND TIMESTAMP 'epoch' + INTERVAL '1 second' * (created * 0.001) <= '2
024-01-01 01:00:00'
ORDER BY created ASC
Traceback (most recent call last):
File "/workspace/scubaduck/scubaduck/server.py", line 617, in query
rows = con.execute(sql).fetchall()
^^^^^^^^^^^^^^^^
duckdb.duckdb.BinderException: Binder Error: No function matches the given name
and argument types '*(INTERVAL, DECIMAL(23,3))'. You might need to add explicit
type casts.
Candidate functions:
*(TINYINT, TINYINT) -> TINYINT
*(SMALLINT, SMALLINT) -> SMALLINT
*(INTEGER, INTEGER) -> INTEGER
*(BIGINT, BIGINT) -> BIGINT
*(HUGEINT, HUGEINT) -> HUGEINT
*(FLOAT, FLOAT) -> FLOAT
*(DOUBLE, DOUBLE) -> DOUBLE
*(DECIMAL, DECIMAL) -> DECIMAL
*(UTINYINT, UTINYINT) -> UTINYINT
*(USMALLINT, USMALLINT) -> USMALLINT
*(UINTEGER, UINTEGER) -> UINTEGER
*(UBIGINT, UBIGINT) -> UBIGINT
*(UHUGEINT, UHUGEINT) -> UHUGEINT
*(INTERVAL, BIGINT) -> INTERVAL
*(BIGINT, INTERVAL) -> INTERVAL
LINE 3: WHERE TIMESTAMP 'epoch' + INTERVAL '1 second' * (created * 0.001) >= '20
24-01-01 00:00:00' AND TIMESTAMP...
^
=========================== short test summary info ============================
FAILED tests/test_server.py::test_integer_time_unit_ms - assert 400 == 200
1 failed, 88 passed in 11.51s
(scubaduck) bash-5.2#The test fails due to invalid multiplication of INTERVAL and DECIMAL in DuckDB. I'll adjust _time_expr to use division by 1000 instead, making it return 'TIMESTAMP 'epoch' + INTERVAL '1 millisecond' * {col}'. I'll check if DuckDB supports 'MILLISECOND' or 'MICROSECOND' and update the expression accordingly.
def _time_expr(col: str, column_types: Dict[str, str] | None, unit: str) -> str:
"""Return SQL expression for column interpreted as timestamp."""
if column_types is None:
return col
ctype = column_types.get(col, "").upper()
if not any(t in ctype for t in ["TIMESTAMP", "DATE", "TIME"]):
if any(
t in ctype
for t in [
"INT",
"DECIMAL",
"REAL",
"DOUBLE",
"FLOAT",
"NUMERIC",
"HUGEINT",
]
):
mult = {
"s": "1",
"ms": "0.001",
"us": "0.000001",(scubaduck) bash-5.2# sed -n '215,230p' scubaduck/server.py
"us": "0.000001",
"ns": "0.000000001",
}.get(unit, "1")
return f"TIMESTAMP 'epoch' + INTERVAL '1 second' * ({col} * {mult})"
return col
def build_query(params: QueryParams, column_types: Dict[str, str] | None = None)
-> str:
select_parts: list[str] = []
group_cols = params.group_by[:]
if params.graph_type == "timeseries":
sec = _granularity_seconds(params.granularity, params.start, params.end)
x_axis = params.x_axis or params.time_column
xexpr = _time_expr(x_axis, column_types, params.time_unit)
if params.start:
bucket_expr = (
(scubaduck) bash-5.2#I'll adjust the code to handle 's' without changes, using the same interval.
> *** Begin Patch
> *** Update File: scubaduck/server.py
> @@
> - mult = {
> - "s": "1",
> - "ms": "0.001",
> - "us": "0.000001",
> - "ns": "0.000000001",
> - }.get(unit, "1")
> - return f"TIMESTAMP 'epoch' + INTERVAL '1 second' * ({col} * {mult
})"
> + interval = {
> + "s": "1 second",
> + "ms": "1 millisecond",
> + "us": "1 microsecond",
> + "ns": "1 nanosecond",
> + }.get(unit, "1 second")
> + return f"TIMESTAMP 'epoch' + INTERVAL '{interval}' * {col}"
> *** End Patch
> PATCH
Success. Updated the following files:
M scubaduck/server.py
(scubaduck) bash-5.2#I'll run ruff format on server.py to apply the changes.
(scubaduck) bash-5.2# ruff format scubaduck/server.py tests/test_server.py tests
/test_web.py >/tmp/format.log && tail -n 20 /tmp/format.log
3 files left unchanged
(scubaduck) bash-5.2#I’ll go ahead and perform a ruff check on the code.
(scubaduck) bash-5.2# ruff check scubaduck/server.py scubaduck/static/index.html
tests/test_server.py tests/test_web.py >/tmp/ruff.log && tail -n 20 /tmp/ruff.l
og
(scubaduck) bash-5.2#I'll try running pyright once more to check for any issues.
(scubaduck) bash-5.2# pyright >/tmp/pyright.log && tail -n 20 /tmp/pyright.logTraceback (most recent call last):
File "/usr/lib/python3.12/urllib/request.py", line 1344, in do_open
h.request(req.get_method(), req.selector, req.data, headers,
File "/usr/lib/python3.12/http/client.py", line 1336, in request
self._send_request(method, url, body, headers, encode_chunked)
File "/usr/lib/python3.12/http/client.py", line 1382, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "/usr/lib/python3.12/http/client.py", line 1331, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "/usr/lib/python3.12/http/client.py", line 1091, in _send_output
self.send(msg)
File "/usr/lib/python3.12/http/client.py", line 1035, in send
self.connect()
File "/usr/lib/python3.12/http/client.py", line 1470, in connect
super().connect()
File "/usr/lib/python3.12/http/client.py", line 1001, in connect
self.sock = self._create_connection(
^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3.12/socket.py", line 852, in create_connection
raise exceptions[0]
File "/usr/lib/python3.12/socket.py", line 837, in create_connection
sock.connect(sa)
OSError: [Errno 113] No route to host
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "<frozen runpy>", line 198, in _run_module_as_main
File "<frozen runpy>", line 88, in _run_code
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/nodeenv.py", lin
e 1548, in <module>
main()
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/nodeenv.py", lin
e 1119, in main
args.node = get_last_stable_node_version()
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/nodeenv.py", lin
e 1052, in get_last_stable_node_version
return _get_versions_json()[0]['version'].lstrip('v')
^^^^^^^^^^^^^^^^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/nodeenv.py", lin
e 1028, in _get_versions_json
response = urlopen('%s/index.json' % src_base_url)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/nodeenv.py", lin
e 652, in urlopen
return urllib2.urlopen(req)
^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3.12/urllib/request.py", line 215, in urlopen
return opener.open(url, data, timeout)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3.12/urllib/request.py", line 515, in open
response = self._open(req, data)
^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3.12/urllib/request.py", line 532, in _open
result = self._call_chain(self.handle_open, protocol, protocol +
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3.12/urllib/request.py", line 492, in _call_chain
result = func(*args)
^^^^^^^^^^^
File "/usr/lib/python3.12/urllib/request.py", line 1392, in https_open
return self.do_open(http.client.HTTPSConnection, req,
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3.12/urllib/request.py", line 1347, in do_open
raise URLError(err)
urllib.error.URLError: <urlopen error [Errno 113] No route to host>
Traceback (most recent call last):
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pyright/node.py"
, line 93, in _install_node_env
subprocess.run(args, check=True)
File "/usr/lib/python3.12/subprocess.py", line 571, in run
raise CalledProcessError(retcode, process.args,
subprocess.CalledProcessError: Command '['/workspace/scubaduck/.venv/bin/python'
, '-m', 'nodeenv', '/root/.cache/pyright-python/nodeenv']' returned non-zero exi
t status 1.
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/workspace/scubaduck/.venv/bin/pyright", line 10, in <module>
sys.exit(entrypoint())
^^^^^^^^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pyright/cli.py",
line 31, in entrypoint
sys.exit(main(sys.argv[1:]))
^^^^^^^^^^^^^^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pyright/cli.py",
line 18, in main
return run(*args, **kwargs).returncode
^^^^^^^^^^^^^^^^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pyright/cli.py",
line 27, in run
return node.run('node', str(script), *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pyright/node.py"
, line 138, in run
strategy = _resolve_strategy(target)
^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pyright/node.py"
, line 130, in _resolve_strategy
return NodeenvStrategy(type='nodeenv', path=_ensure_node_env(target))
^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pyright/node.py"
, line 54, in _ensure_node_env
_install_node_env()
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pyright/node.py"
, line 95, in _install_node_env
raise RuntimeError(
RuntimeError: nodeenv failed; for more reliable node.js binaries try `pip instal
l pyright[nodejs]`
(scubaduck) bash-5.2#Pyright failed because of a network issue. I'll show /tmp/pyright.log for details.
(scubaduck) bash-5.2# tail -n 20 /tmp/pyright.log
(scubaduck) bash-5.2#The log is empty from the earlier failure. I'll rerun tests after fixing.
(scubaduck) bash-5.2# pytest tests/test_server.py::test_integer_time_unit_ms -q
>/tmp/test_single.log && cat /tmp/test_single.logbringing up nodes...
bringing up nodes...
F [100%]
=================================== FAILURES ===================================
__________________________ test_integer_time_unit_ms ___________________________
[gw0] linux -- Python 3.12.3 /workspace/scubaduck/.venv/bin/python
Traceback (most recent call last):
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/runner.p
y", line 341, in from_call
result: TResult | None = func()
^^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/runner.p
y", line 242, in <lambda>
lambda: runtest_hook(item=item, **kwds), when=when, reraise=reraise
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_hooks.py
", line 512, in __call__
return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult
)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_manager.
py", line 120, in _hookexec
return self._inner_hookexec(hook_name, methods, kwargs, firstresult)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_callers.
py", line 167, in _multicall
raise exception
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_callers.
py", line 139, in _multicall
teardown.throw(exception)
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/threadex
ception.py", line 92, in pytest_runtest_call
yield from thread_exception_runtest_hook()
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/threadex
ception.py", line 68, in thread_exception_runtest_hook
yield
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_callers.
py", line 139, in _multicall
teardown.throw(exception)
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/unraisab
leexception.py", line 95, in pytest_runtest_call
yield from unraisable_exception_runtest_hook()
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/unraisab
leexception.py", line 70, in unraisable_exception_runtest_hook
yield
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_callers.
py", line 139, in _multicall
teardown.throw(exception)
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/logging.
py", line 846, in pytest_runtest_call
yield from self._runtest_for(item, "call")
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/logging.
py", line 829, in _runtest_for
yield
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_callers.
py", line 139, in _multicall
teardown.throw(exception)
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/capture.
py", line 898, in pytest_runtest_call
return (yield)
^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_callers.
py", line 139, in _multicall
teardown.throw(exception)
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/skipping
.py", line 257, in pytest_runtest_call
return (yield)
^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_callers.
py", line 121, in _multicall
res = hook_impl.function(*args)
^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/runner.p
y", line 174, in pytest_runtest_call
item.runtest()
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/python.p
y", line 1627, in runtest
self.ihook.pytest_pyfunc_call(pyfuncitem=self)
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_hooks.py
", line 512, in __call__
return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult
)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_manager.
py", line 120, in _hookexec
return self._inner_hookexec(hook_name, methods, kwargs, firstresult)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_callers.
py", line 167, in _multicall
raise exception
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_callers.
py", line 121, in _multicall
res = hook_impl.function(*args)
^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/python.p
y", line 159, in pytest_pyfunc_call
result = testfunction(**testargs)
^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/scubaduck/tests/test_server.py", line 358, in test_integer_ti
me_unit_ms
assert rv.status_code == 200
AssertionError: assert 400 == 200
+ where 400 = <WrapperTestResponse 755 bytes [400 BAD REQUEST]>.status_code
----------------------------- Captured stdout call -----------------------------
Query failed:
SELECT created, event
FROM "events"
WHERE TIMESTAMP 'epoch' + INTERVAL '1 millisecond' * created >= '2024-01-01 00:0
0:00' AND TIMESTAMP 'epoch' + INTERVAL '1 millisecond' * created <= '2024-01-01
01:00:00'
ORDER BY created ASC
Traceback (most recent call last):
File "/workspace/scubaduck/scubaduck/server.py", line 617, in query
rows = con.execute(sql).fetchall()
^^^^^^^^^^^^^^^^
duckdb.duckdb.InvalidInputException: Invalid Input Error: Type INT64 with value
1704067200000 can't be cast because the value is out of range for the destinatio
n type INT32
=========================== short test summary info ============================
FAILED tests/test_server.py::test_integer_time_unit_ms - assert 400 == 200
1 failed in 0.64s
(scubaduck) bash-5.2#To avoid overflow, I'll cast columns to BIGINT. Time expressions need adjustments too.
(scubaduck) bash-5.2# apply_patch<<'PATCH'
> *** Begin Patch
> *** Update File: scubaduck/server.py
> @@
> - interval = {
> - "s": "1 second",
> - "ms": "1 millisecond",
> - "us": "1 microsecond",
> - "ns": "1 nanosecond",
> - }.get(unit, "1 second")
> - return f"TIMESTAMP 'epoch' + INTERVAL '{interval}' * {col}"
> + interval = {
> + "s": "1 second",
> + "ms": "1 millisecond",
> + "us": "1 microsecond",
> + "ns": "1 nanosecond",
> + }.get(unit, "1 second")
> + return (
> + f"TIMESTAMP 'epoch' + INTERVAL '{interval}' * CAST({col} AS B
IGINT)"
> + )
> *** End Patch
> PATCH
Success. Updated the following files:
M scubaduck/server.py(scubaduck) bash-5.2# ruff format scubaduck/server.py tests/test_server.py tests
/test_web.py >/tmp/format.log && tail -n 20 /tmp/format.log
1 file reformatted, 2 files left unchanged
(scubaduck) bash-5.2#I'm checking the code using ruff now.
(scubaduck) bash-5.2# ruff check scubaduck/server.py scubaduck/static/index.html
tests/test_server.py tests/test_web.py >/tmp/ruff.log && tail -n 20 /tmp/ruff.l
og
(scubaduck) bash-5.2#Okay, time to give that test another shot.
(scubaduck) bash-5.2# pytest tests/test_server.py::test_integer_time_unit_ms -q
>/tmp/test_single.log && cat /tmp/test_single.logbringing up nodes...
bringing up nodes...
F [100%]
=================================== FAILURES ===================================
__________________________ test_integer_time_unit_ms ___________________________
[gw0] linux -- Python 3.12.3 /workspace/scubaduck/.venv/bin/python
Traceback (most recent call last):
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/runner.p
y", line 341, in from_call
result: TResult | None = func()
^^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/runner.p
y", line 242, in <lambda>
lambda: runtest_hook(item=item, **kwds), when=when, reraise=reraise
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_hooks.py
", line 512, in __call__
return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult
)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_manager.
py", line 120, in _hookexec
return self._inner_hookexec(hook_name, methods, kwargs, firstresult)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_callers.
py", line 167, in _multicall
raise exception
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_callers.
py", line 139, in _multicall
teardown.throw(exception)
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/threadex
ception.py", line 92, in pytest_runtest_call
yield from thread_exception_runtest_hook()
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/threadex
ception.py", line 68, in thread_exception_runtest_hook
yield
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_callers.
py", line 139, in _multicall
teardown.throw(exception)
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/unraisab
leexception.py", line 95, in pytest_runtest_call
yield from unraisable_exception_runtest_hook()
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/unraisab
leexception.py", line 70, in unraisable_exception_runtest_hook
yield
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_callers.
py", line 139, in _multicall
teardown.throw(exception)
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/logging.
py", line 846, in pytest_runtest_call
yield from self._runtest_for(item, "call")
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/logging.
py", line 829, in _runtest_for
yield
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_callers.
py", line 139, in _multicall
teardown.throw(exception)
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/capture.
py", line 898, in pytest_runtest_call
return (yield)
^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_callers.
py", line 139, in _multicall
teardown.throw(exception)
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/skipping
.py", line 257, in pytest_runtest_call
return (yield)
^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_callers.
py", line 121, in _multicall
res = hook_impl.function(*args)
^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/runner.p
y", line 174, in pytest_runtest_call
item.runtest()
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/python.p
y", line 1627, in runtest
self.ihook.pytest_pyfunc_call(pyfuncitem=self)
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_hooks.py
", line 512, in __call__
return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult
)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_manager.
py", line 120, in _hookexec
return self._inner_hookexec(hook_name, methods, kwargs, firstresult)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_callers.
py", line 167, in _multicall
raise exception
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_callers.
py", line 121, in _multicall
res = hook_impl.function(*args)
^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/python.p
y", line 159, in pytest_pyfunc_call
result = testfunction(**testargs)
^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/scubaduck/tests/test_server.py", line 358, in test_integer_ti
me_unit_ms
assert rv.status_code == 200
AssertionError: assert 400 == 200
+ where 400 = <WrapperTestResponse 787 bytes [400 BAD REQUEST]>.status_code
----------------------------- Captured stdout call -----------------------------
Query failed:
SELECT created, event
FROM "events"
WHERE TIMESTAMP 'epoch' + INTERVAL '1 millisecond' * CAST(created AS BIGINT) >=
'2024-01-01 00:00:00' AND TIMESTAMP 'epoch' + INTERVAL '1 millisecond' * CAST(cr
eated AS BIGINT) <= '2024-01-01 01:00:00'
ORDER BY created ASC
Traceback (most recent call last):
File "/workspace/scubaduck/scubaduck/server.py", line 617, in query
rows = con.execute(sql).fetchall()
^^^^^^^^^^^^^^^^
duckdb.duckdb.InvalidInputException: Invalid Input Error: Type INT64 with value
1704067200000 can't be cast because the value is out of range for the destinatio
n type INT32
=========================== short test summary info ============================
FAILED tests/test_server.py::test_integer_time_unit_ms - assert 400 == 200
1 failed in 0.65s
(scubaduck) bash-5.2#It seems DuckDB is interpreting the CSV value as INT64, causing casting failure during query execution.
The issue might be due to INT32 typing in start/end; let's test by querying directly.
(scubaduck) bash-5.2# python - <<'PY'
> import duckdb, pandas as pd
> import io
> import sys
> csv='created,event\n1704067200000,login\n1704070800000,logout\n'
> con=duckdb.connect()
> con.execute("CREATE TABLE events AS SELECT * FROM read_csv_auto('events.csv')"
)
> PY
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
ModuleNotFoundError: No module named 'pandas'
(scubaduck) bash-5.2#I’ll work directly with the CSV path since pandas isn’t available in this environment.
(scubaduck) bash-5.2# python - <<'PY'
> import duckdb, os
> open('tmp.csv','w').write('created,event\n1704067200000,login\n1704070800000,l
ogout\n')
> con=duckdb.connect()
> con.execute("CREATE TABLE events AS SELECT * FROM read_csv_auto('tmp.csv')")
> print(con.execute('PRAGMA table_info(events)').fetchall())
> rows=con.execute("SELECT created FROM events").fetchall()
> print(rows)
> PY
[(0, 'created', 'BIGINT', False, None, False), (1, 'event', 'VARCHAR', False, No
ne, False)]
[(1704067200000,), (1704070800000,)]
(scubaduck) bash-5.2#I'm checking if the INT32 error might be tied to casting from Python to SQL.
The error is likely from a millisecond interval exceeding INT32 limits during casting.
Dividing by 1000 and using fractional seconds should keep the value within INT32 limits.
(scubaduck) bash-5.2# python - <<'PY'
> import duckdb
> con=duckdb.connect()
> con.execute("SELECT TIMESTAMP 'epoch' + INTERVAL '1 second' * (1704067200000 /
1000.0)")
> print(con.fetchall())
> PY
Traceback (most recent call last):
File "<stdin>", line 3, in <module>
duckdb.duckdb.BinderException: Binder Error: No function matches the given name
and argument types '*(INTERVAL, DOUBLE)'. You might need to add explicit type ca
sts.
Candidate functions:
*(TINYINT, TINYINT) -> TINYINT
*(SMALLINT, SMALLINT) -> SMALLINT
*(INTEGER, INTEGER) -> INTEGER
*(BIGINT, BIGINT) -> BIGINT
*(HUGEINT, HUGEINT) -> HUGEINT
*(FLOAT, FLOAT) -> FLOAT
*(DOUBLE, DOUBLE) -> DOUBLE
*(DECIMAL, DECIMAL) -> DECIMAL
*(UTINYINT, UTINYINT) -> UTINYINT
*(USMALLINT, USMALLINT) -> USMALLINT
*(UINTEGER, UINTEGER) -> UINTEGER
*(UBIGINT, UBIGINT) -> UBIGINT
*(UHUGEINT, UHUGEINT) -> UHUGEINT
*(INTERVAL, BIGINT) -> INTERVAL
*(BIGINT, INTERVAL) -> INTERVAL
LINE 1: SELECT TIMESTAMP 'epoch' + INTERVAL '1 second' * (1704067200000 / 1000.0
)
^
(scubaduck) bash-5.2#It seems I need to cast to BIGINT, but I'll double-check if that's correct.
(scubaduck) bash-5.2# python - <<'PY'
> import duckdb
> con=duckdb.connect()
> con.execute("SELECT TIMESTAMP 'epoch' + INTERVAL '1 second' * CAST(17040672000
00 / 1000 AS BIGINT)")
> print(con.fetchall())
> PY
[(datetime.datetime(2024, 1, 1, 0, 0),)]
(scubaduck) bash-5.2#I'll patch _time_expr, updating the formula based on the unit and divisor.
def _time_expr(col: str, column_types: Dict[str, str] | None, unit: str) -> str:
"""Return SQL expression for column interpreted as timestamp."""
if column_types is None:
return col
ctype = column_types.get(col, "").upper()
if not any(t in ctype for t in ["TIMESTAMP", "DATE", "TIME"]):
if any(
t in ctype
for t in [
"INT",
"DECIMAL",
"REAL",
"DOUBLE",
"FLOAT",
"NUMERIC",
"HUGEINT",
]
):
interval = {
"s": "1 second",
"ms": "1 millisecond",
"us": "1 microsecond",
"ns": "1 nanosecond",
}.get(unit, "1 second")
return f"TIMESTAMP 'epoch' + INTERVAL '{interval}' * CAST({col} AS B
IGINT)"
return col
def build_query(params: QueryParams, column_types: Dict[str, str] | None = None)
-> str:
select_parts: list[str] = []
group_cols = params.group_by[:]
if params.graph_type == "timeseries":
sec = _granularity_seconds(params.granularity, params.start, params.end)
x_axis = params.x_axis or params.time_column
xexpr = _time_expr(x_axis, column_types, params.time_unit)
if params.start:
bucket_expr = (
(scubaduck) bash-5.2#I'm updating things to reflect the new formula.
> *** Begin Patch
> *** Update File: scubaduck/server.py
> @@
> - interval = {
> - "s": "1 second",
> - "ms": "1 millisecond",
> - "us": "1 microsecond",
> - "ns": "1 nanosecond",
> - }.get(unit, "1 second")
> - return f"TIMESTAMP 'epoch' + INTERVAL '{interval}' * CAST({col} A
S BIGINT)"
> + divisor = {
> + "s": 1,
> + "ms": 1000,
> + "us": 1_000_000,
> + "ns": 1_000_000_000,
> + }.get(unit, 1)
> + if divisor == 1:
> + expr = f"CAST({col} AS BIGINT)"
> + else:
> + expr = f"CAST({col} / {divisor} AS BIGINT)"
> + return f"TIMESTAMP 'epoch' + INTERVAL '1 second' * {expr}"
> *** End Patch
> PATCH
Success. Updated the following files:
M scubaduck/server.py(scubaduck) bash-5.2# ruff format scubaduck/server.py tests/test_server.py tests
/test_web.py >/tmp/format.log && tail -n 20 /tmp/format.log
3 files left unchanged
(scubaduck) bash-5.2#I'll run a quick ruff check to catch any issues in the code.
(scubaduck) bash-5.2# ruff check scubaduck/server.py scubaduck/static/index.html
tests/test_server.py tests/test_web.py >/tmp/ruff.log && tail -n 20 /tmp/ruff.l
og
(scubaduck) bash-5.2#I'll try running the test one more time.
(scubaduck) bash-5.2# pytest tests/test_server.py::test_integer_time_unit_ms -q
>/tmp/test_single.log && cat /tmp/test_single.logbringing up nodes...
bringing up nodes...
. [100%]
1 passed in 0.63s
(scubaduck) bash-5.2#Time to execute the full test suite and check everything works as expected.
(scubaduck) bash-5.2# pytest -q >/tmp/pytest.log && tail -n 20 /tmp/pytest.logbringing up nodes...
bringing up nodes...
........................................................................ [ 80%]
................. [100%]
89 passed in 11.08s
(scubaduck) bash-5.2#All tests have passed, so I'll go ahead and commit the changes.
(scubaduck) bash-5.2# git status --short
M scubaduck/server.py
M scubaduck/static/index.html
M tests/test_server.py
M tests/test_web.py
?? tmp.csv
(scubaduck) bash-5.2#I’ll remove the leftover tmp.csv from the manual test.
(scubaduck) bash-5.2# rm tmp.csv(scubaduck) bash-5.2# git status --short
M scubaduck/server.py
M scubaduck/static/index.html
M tests/test_server.py
M tests/test_web.py
(scubaduck) bash-5.2#It's time to commit the updates I've made!
sts/test_server.py tests/test_web.py(scubaduck) bash-5.2# git commit -m "feat(ui): add time unit dropdown" && git st
atus --short
[work 97a0de1] feat(ui): add time unit dropdown
4 files changed, 62 insertions(+), 5 deletions(-)
(scubaduck) bash-5.2#I should check the git status to confirm how many modifications are present.
(scubaduck) bash-5.2#I'll display the snippet lines for clarity, especially the server file patch lines.
(scubaduck) bash-5.2# nl -ba scubaduck/server.py | sed -n '186,242p'
186 except Exception:
187 return 3600
188 total = max((e - s).total_seconds(), 1)
189 buckets = 100 if gran == "auto" else 500
190 return max(int(total // buckets), 1)
191 return 3600
192
193
194 def _time_expr(col: str, column_types: Dict[str, str] | None, unit: str)
-> str:
195 """Return SQL expression for column interpreted as timestamp."""
196 if column_types is None:
197 return col
198 ctype = column_types.get(col, "").upper()
199 if not any(t in ctype for t in ["TIMESTAMP", "DATE", "TIME"]):
200 if any(
201 t in ctype
202 for t in [
203 "INT",
204 "DECIMAL",
205 "REAL",
206 "DOUBLE",
207 "FLOAT",
208 "NUMERIC",
209 "HUGEINT",
210 ]
211 ):
212 divisor = {
213 "s": 1,
214 "ms": 1000,
215 "us": 1_000_000,
216 "ns": 1_000_000_000,
217 }.get(unit, 1)
218 if divisor == 1:
219 expr = f"CAST({col} AS BIGINT)"
220 else:
221 expr = f"CAST({col} / {divisor} AS BIGINT)"
222 return f"TIMESTAMP 'epoch' + INTERVAL '1 second' * {expr}"
223 return col
224
225
226 def build_query(params: QueryParams, column_types: Dict[str, str] | None
= None) -> str:
227 select_parts: list[str] = []
228 group_cols = params.group_by[:]
229 if params.graph_type == "timeseries":
230 sec = _granularity_seconds(params.granularity, params.start, par
ams.end)
231 x_axis = params.x_axis or params.time_column
232 xexpr = _time_expr(x_axis, column_types, params.time_unit)
233 if params.start:
234 bucket_expr = (
235 f"TIMESTAMP '{params.start}' + INTERVAL '{sec} second' *
"
236 f"CAST(floor((epoch({xexpr}) - epoch(TIMESTAMP '{params.
start}'))/{sec}) AS BIGINT)"
237 )
238 else:
239 bucket_expr = (
240 f"TIMESTAMP 'epoch' + INTERVAL '{sec} second' * "
241 f"CAST(floor(epoch({xexpr})/{sec}) AS BIGINT)"
242 )(scubaduck) bash-5.2# nl -ba scubaduck/server.py | sed -n '20,50p'
20
21 @dataclass
22 class Filter:
23 column: str
24 op: str
25 value: str | int | float | list[str] | None
26
27
28 @dataclass
29 class QueryParams:
30 start: str | None = None
31 end: str | None = None
32 order_by: str | None = None
33 order_dir: str = "ASC"
34 limit: int | None = None
35 columns: list[str] = field(default_factory=lambda: [])
36 filters: list[Filter] = field(default_factory=lambda: [])
37 derived_columns: dict[str, str] = field(default_factory=lambda: {})
38 graph_type: str = "samples"
39 group_by: list[str] = field(default_factory=lambda: [])
40 aggregate: str | None = None
41 show_hits: bool = False
42 x_axis: str | None = None
43 granularity: str = "Auto"
44 fill: str = "0"
45 table: str = "events"
46 time_column: str = "timestamp"
47 time_unit: str = "s"
48
49
50 def _normalize_sqlite_type(sql: str) -> str:
(scubaduck) bash-5.2#I'll write a simple snippet to implement the new dropdown in HTML.
(scubaduck) bash-5.2# nl -ba scubaduck/static/index.html | sed -n '188,206p'
188 <button class="tab active" data-tab="settings">View Settings</bu
tton>
189 <button class="tab" data-tab="columns">Columns</button>
190 <button id="dive" onclick="dive()">Dive</button>
191 </div>
192 <div id="settings" class="tab-content active">
193 <div class="field">
194 <label>Time Column</label>
195 <select id="time_column"></select>
196 <select id="time_unit" style="margin-left:4px">
197 <option value="s">s</option>
198 <option value="ms">ms</option>
199 <option value="us">us</option>
200 <option value="ns">ns</option>
201 </select>
202 </div>
203 <div class="field">
204 <label>Start<span class="help" title="Sets the start/end of th
e time range to query. Can be any kind of datetime string. For example: 'April 2
3, 2014' or 'yesterday'.">[?]</span></label>
205 <div class="rel-box">
206 <input id="start" type="text" />
(scubaduck) bash-5.2#I'll create a new JS snippet to incorporate time_unit for formatting.
(scubaduck) bash-5.2# nl -ba scubaduck/static/index.html | sed -n '1168,1190p'
1168 rows.forEach(row => {
1169 const tr = document.createElement('tr');
1170 tr.addEventListener('click', () => {
1171 const wasSelected = tr.classList.contains('selected');
1172 document
1173 .querySelectorAll('#results tr.selected')
1174 .forEach(el => el.classList.remove('selected'));
1175 if (!wasSelected) {
1176 tr.classList.add('selected');
1177 }
1178 });
1179 row.forEach((v, i) => {
1180 const col = selectedColumns[i];
1181 const td = document.createElement('td');
1182 if (isTimeColumn(col)) {
1183 let d;
1184 const t = (columnTypes[col] || '').toUpperCase();
1185 if (t.includes('TIMESTAMP') || t.includes('DATE') || t.includes(
'TIME')) {
1186 d = new Date(v);
1187 } else {
1188 const unit = document.getElementById('time_unit').value;
1189 const factors = {s: 1000, ms: 1, us: 0.001, ns: 0.000001};
1190 d = new Date(Number(v) * (factors[unit] || 1000));
(scubaduck) bash-5.2#I'll create the PR summary, highlighting the patch lines and including pytest results.
(scubaduck) bash-5.2# nl -ba tests/test_server.py | sed -n '318,364p'
318
319 def test_integer_time_column(tmp_path: Path) -> None:
320 csv_file = tmp_path / "events.csv"
321 csv_file.write_text("created,event\n1704067200,login\n1704070800,log
out\n")
322 app = server.create_app(csv_file)
323 client = app.test_client()
324 payload = {
325 "table": "events",
326 "start": "2024-01-01 00:00:00",
327 "end": "2024-01-01 01:00:00",
328 "order_by": "created",
329 "columns": ["created", "event"],
330 "time_column": "created",
331 }
332 rv = client.post(
333 "/api/query", data=json.dumps(payload), content_type="applicatio
n/json"
334 )
335 data = rv.get_json()
336 assert rv.status_code == 200
337 assert len(data["rows"]) == 2
338
339
340 def test_integer_time_unit_ms(tmp_path: Path) -> None:
341 csv_file = tmp_path / "events.csv"
342 csv_file.write_text("created,event\n1704067200000,login\n17040708000
00,logout\n")
343 app = server.create_app(csv_file)
344 client = app.test_client()
345 payload = {
346 "table": "events",
347 "start": "2024-01-01 00:00:00",
348 "end": "2024-01-01 01:00:00",
349 "order_by": "created",
350 "columns": ["created", "event"],
351 "time_column": "created",
352 "time_unit": "ms",
353 }
354 rv = client.post(
355 "/api/query", data=json.dumps(payload), content_type="applicatio
n/json"
356 )
357 data = rv.get_json()
358 assert rv.status_code == 200
359 assert len(data["rows"]) == 2
360
361
362 def test_envvar_db(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> N
one:
363 csv_file = tmp_path / "custom.csv"
364 csv_file.write_text("timestamp,event,value,user\n2024-01-01 00:00:00
,login,5,bob\n")
(scubaduck) bash-5.2#I'll grab the lines for tests/test_web now.
(scubaduck) bash-5.2# nl -ba tests/test_web.py | sed -n '96,114p'
96 assert len(data["rows"]) == 2
97
98
99 def test_time_column_dropdown(page: Any, server_url: str) -> None:
100 page.goto(server_url)
101 page.wait_for_selector("#time_column option", state="attached")
102 options = page.locator("#time_column option").all_inner_texts()
103 assert "timestamp" in options
104 assert "value" in options
105 assert page.input_value("#time_column") == "timestamp"
106
107
108 def test_time_unit_dropdown(page: Any, server_url: str) -> None:
109 page.goto(server_url)
110 page.wait_for_selector("#time_unit", state="attached")
111 opts = page.locator("#time_unit option").all_inner_texts()
112 assert "ms" in opts
113 assert page.input_value("#time_unit") == "s"
114
(scubaduck) bash-5.2#I'll pull the test output lines and then create the PR.