Task ID: task_e_682e1c53a6108323b91598d5c4d9e9c8
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (5/33)
execnet ------------------------------ 30.87 KiB/39.66 KiB
pytest-xdist ------------------------------ 45.03 KiB/45.03 KiB
requests ------------------------------ 63.41 KiB/63.41 KiB
packaging ------------------------------ 64.91 KiB/64.91 KiB
idna ------------------------------ 48.00 KiB/68.79 KiB
text-unidecode ------------------------------ 76.32 KiB/76.32 KiB
click ------------------------------ 64.00 KiB/99.76 KiB
flask ------------------------------ 64.00 KiB/100.88 KiB
urllib3 ------------------------------ 78.21 KiB/125.66 KiB
jinja2 ------------------------------ 112.00 KiB/131.74 KiB
charset-normalizer ------------------------------ 145.08 KiB/145.08 KiB
certifi ------------------------------ 155.88 KiB/155.88 KiB
werkzeug ------------------------------ 142.18 KiB/219.24 KiB
python-dateutil ------------------------------ 80.00 KiB/224.50 KiB
pytest ------------------------------ 107.88 KiB/335.58 KiB
greenlet ------------------------------ 156.96 KiB/589.71 KiB
pyright ------------------------------ 16.00 KiB/5.31 MiB
ruff ------------------------------ 159.19 KiB/11.02 MiB
duckdb ------------------------------ 320.00 KiB/19.27 MiB
playwright ------------------------------ 32.00 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (5/33)
execnet ------------------------------ 30.87 KiB/39.66 KiB
pytest-xdist ------------------------------ 45.03 KiB/45.03 KiB
requests ------------------------------ 63.41 KiB/63.41 KiB
packaging ------------------------------ 64.91 KiB/64.91 KiB
idna ------------------------------ 48.00 KiB/68.79 KiB
click ------------------------------ 64.00 KiB/99.76 KiB
flask ------------------------------ 64.00 KiB/100.88 KiB
urllib3 ------------------------------ 78.21 KiB/125.66 KiB
jinja2 ------------------------------ 112.00 KiB/131.74 KiB
charset-normalizer ------------------------------ 145.08 KiB/145.08 KiB
certifi ------------------------------ 155.88 KiB/155.88 KiB
werkzeug ------------------------------ 142.18 KiB/219.24 KiB
python-dateutil ------------------------------ 80.00 KiB/224.50 KiB
pytest ------------------------------ 107.88 KiB/335.58 KiB
greenlet ------------------------------ 156.96 KiB/589.71 KiB
pyright ------------------------------ 16.00 KiB/5.31 MiB
ruff ------------------------------ 159.19 KiB/11.02 MiB
duckdb ------------------------------ 320.00 KiB/19.27 MiB
playwright ------------------------------ 32.00 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (5/33)
execnet ------------------------------ 30.87 KiB/39.66 KiB
pytest-xdist ------------------------------ 45.03 KiB/45.03 KiB
requests ------------------------------ 63.41 KiB/63.41 KiB
packaging ------------------------------ 64.91 KiB/64.91 KiB
idna ------------------------------ 64.00 KiB/68.79 KiB
click ------------------------------ 80.00 KiB/99.76 KiB
flask ------------------------------ 64.00 KiB/100.88 KiB
urllib3 ------------------------------ 94.21 KiB/125.66 KiB
jinja2 ------------------------------ 128.00 KiB/131.74 KiB
charset-normalizer ------------------------------ 145.08 KiB/145.08 KiB
werkzeug ------------------------------ 158.18 KiB/219.24 KiB
python-dateutil ------------------------------ 144.00 KiB/224.50 KiB
pytest ------------------------------ 123.88 KiB/335.58 KiB
greenlet ------------------------------ 220.96 KiB/589.71 KiB
pyright ------------------------------ 16.00 KiB/5.31 MiB
ruff ------------------------------ 223.19 KiB/11.02 MiB
duckdb ------------------------------ 400.00 KiB/19.27 MiB
playwright ------------------------------ 48.00 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (5/33)
execnet ------------------------------ 30.87 KiB/39.66 KiB
pytest-xdist ------------------------------ 45.03 KiB/45.03 KiB
requests ------------------------------ 63.41 KiB/63.41 KiB
packaging ------------------------------ 64.91 KiB/64.91 KiB
idna ------------------------------ 64.00 KiB/68.79 KiB
click ------------------------------ 80.00 KiB/99.76 KiB
flask ------------------------------ 64.00 KiB/100.88 KiB
urllib3 ------------------------------ 94.21 KiB/125.66 KiB
jinja2 ------------------------------ 128.00 KiB/131.74 KiB
charset-normalizer ------------------------------ 145.08 KiB/145.08 KiB
werkzeug ------------------------------ 158.18 KiB/219.24 KiB
python-dateutil ------------------------------ 176.00 KiB/224.50 KiB
pytest ------------------------------ 123.88 KiB/335.58 KiB
greenlet ------------------------------ 252.96 KiB/589.71 KiB
pyright ------------------------------ 16.00 KiB/5.31 MiB
ruff ------------------------------ 255.19 KiB/11.02 MiB
duckdb ------------------------------ 416.00 KiB/19.27 MiB
playwright ------------------------------ 48.00 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (5/33)
execnet ------------------------------ 39.66 KiB/39.66 KiB
pytest-xdist ------------------------------ 45.03 KiB/45.03 KiB
packaging ------------------------------ 64.91 KiB/64.91 KiB
idna ------------------------------ 68.79 KiB/68.79 KiB
click ------------------------------ 80.00 KiB/99.76 KiB
flask ------------------------------ 80.00 KiB/100.88 KiB
urllib3 ------------------------------ 94.21 KiB/125.66 KiB
jinja2 ------------------------------ 128.00 KiB/131.74 KiB
charset-normalizer ------------------------------ 145.08 KiB/145.08 KiB
werkzeug ------------------------------ 158.18 KiB/219.24 KiB
python-dateutil ------------------------------ 208.00 KiB/224.50 KiB
pytest ------------------------------ 139.88 KiB/335.58 KiB
greenlet ------------------------------ 316.96 KiB/589.71 KiB
pyright ------------------------------ 16.00 KiB/5.31 MiB
ruff ------------------------------ 303.19 KiB/11.02 MiB
duckdb ------------------------------ 464.00 KiB/19.27 MiB
playwright ------------------------------ 48.00 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (5/33)
execnet ------------------------------ 39.66 KiB/39.66 KiB
pytest-xdist ------------------------------ 45.03 KiB/45.03 KiB
packaging ------------------------------ 64.91 KiB/64.91 KiB
click ------------------------------ 96.00 KiB/99.76 KiB
flask ------------------------------ 80.00 KiB/100.88 KiB
urllib3 ------------------------------ 94.21 KiB/125.66 KiB
jinja2 ------------------------------ 128.00 KiB/131.74 KiB
charset-normalizer ------------------------------ 145.08 KiB/145.08 KiB
werkzeug ------------------------------ 158.18 KiB/219.24 KiB
python-dateutil ------------------------------ 224.50 KiB/224.50 KiB
pytest ------------------------------ 139.88 KiB/335.58 KiB
greenlet ------------------------------ 332.96 KiB/589.71 KiB
pyright ------------------------------ 16.00 KiB/5.31 MiB
ruff ------------------------------ 335.19 KiB/11.02 MiB
duckdb ------------------------------ 480.00 KiB/19.27 MiB
playwright ------------------------------ 48.00 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (5/33)
execnet ------------------------------ 39.66 KiB/39.66 KiB
packaging ------------------------------ 64.91 KiB/64.91 KiB
click ------------------------------ 96.00 KiB/99.76 KiB
flask ------------------------------ 80.00 KiB/100.88 KiB
urllib3 ------------------------------ 94.21 KiB/125.66 KiB
jinja2 ------------------------------ 128.00 KiB/131.74 KiB
charset-normalizer ------------------------------ 145.08 KiB/145.08 KiB
werkzeug ------------------------------ 158.18 KiB/219.24 KiB
python-dateutil ------------------------------ 224.50 KiB/224.50 KiB
pytest ------------------------------ 139.88 KiB/335.58 KiB
greenlet ------------------------------ 348.96 KiB/589.71 KiB
pyright ------------------------------ 16.00 KiB/5.31 MiB
ruff ------------------------------ 335.19 KiB/11.02 MiB
duckdb ------------------------------ 496.00 KiB/19.27 MiB
playwright ------------------------------ 48.00 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (5/33)
execnet ------------------------------ 39.66 KiB/39.66 KiB
packaging ------------------------------ 64.91 KiB/64.91 KiB
click ------------------------------ 96.00 KiB/99.76 KiB
flask ------------------------------ 80.00 KiB/100.88 KiB
urllib3 ------------------------------ 94.21 KiB/125.66 KiB
jinja2 ------------------------------ 128.00 KiB/131.74 KiB
werkzeug ------------------------------ 158.18 KiB/219.24 KiB
python-dateutil ------------------------------ 224.50 KiB/224.50 KiB
pytest ------------------------------ 155.88 KiB/335.58 KiB
greenlet ------------------------------ 364.96 KiB/589.71 KiB
pyright ------------------------------ 16.00 KiB/5.31 MiB
ruff ------------------------------ 367.19 KiB/11.02 MiB
duckdb ------------------------------ 528.00 KiB/19.27 MiB
playwright ------------------------------ 48.00 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (5/33)
execnet ------------------------------ 39.66 KiB/39.66 KiB
click ------------------------------ 96.00 KiB/99.76 KiB
flask ------------------------------ 80.00 KiB/100.88 KiB
urllib3 ------------------------------ 94.21 KiB/125.66 KiB
jinja2 ------------------------------ 128.00 KiB/131.74 KiB
werkzeug ------------------------------ 158.18 KiB/219.24 KiB
python-dateutil ------------------------------ 224.50 KiB/224.50 KiB
pytest ------------------------------ 155.88 KiB/335.58 KiB
greenlet ------------------------------ 364.96 KiB/589.71 KiB
pyright ------------------------------ 16.00 KiB/5.31 MiB
ruff ------------------------------ 367.19 KiB/11.02 MiB
duckdb ------------------------------ 528.00 KiB/19.27 MiB
playwright ------------------------------ 48.00 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (5/33)
execnet ------------------------------ 39.66 KiB/39.66 KiB
click ------------------------------ 99.76 KiB/99.76 KiB
flask ------------------------------ 80.00 KiB/100.88 KiB
urllib3 ------------------------------ 94.21 KiB/125.66 KiB
werkzeug ------------------------------ 174.18 KiB/219.24 KiB
python-dateutil ------------------------------ 224.50 KiB/224.50 KiB
pytest ------------------------------ 171.88 KiB/335.58 KiB
greenlet ------------------------------ 428.96 KiB/589.71 KiB
pyright ------------------------------ 62.16 KiB/5.31 MiB
ruff ------------------------------ 431.19 KiB/11.02 MiB
duckdb ------------------------------ 592.00 KiB/19.27 MiB
playwright ------------------------------ 64.00 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (5/33)
execnet ------------------------------ 39.66 KiB/39.66 KiB
flask ------------------------------ 96.00 KiB/100.88 KiB
urllib3 ------------------------------ 110.21 KiB/125.66 KiB
werkzeug ------------------------------ 190.18 KiB/219.24 KiB
python-dateutil ------------------------------ 224.50 KiB/224.50 KiB
pytest ------------------------------ 187.88 KiB/335.58 KiB
greenlet ------------------------------ 460.96 KiB/589.71 KiB
pyright ------------------------------ 126.16 KiB/5.31 MiB
ruff ------------------------------ 495.19 KiB/11.02 MiB
duckdb ------------------------------ 656.00 KiB/19.27 MiB
playwright ------------------------------ 64.00 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (5/33)
execnet ------------------------------ 39.66 KiB/39.66 KiB
flask ------------------------------ 96.00 KiB/100.88 KiB
urllib3 ------------------------------ 110.21 KiB/125.66 KiB
werkzeug ------------------------------ 190.18 KiB/219.24 KiB
pytest ------------------------------ 187.88 KiB/335.58 KiB
greenlet ------------------------------ 460.96 KiB/589.71 KiB
pyright ------------------------------ 158.16 KiB/5.31 MiB
ruff ------------------------------ 527.19 KiB/11.02 MiB
duckdb ------------------------------ 688.00 KiB/19.27 MiB
playwright ------------------------------ 64.00 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (5/33)
flask ------------------------------ 96.00 KiB/100.88 KiB
urllib3 ------------------------------ 110.21 KiB/125.66 KiB
werkzeug ------------------------------ 190.18 KiB/219.24 KiB
pytest ------------------------------ 187.88 KiB/335.58 KiB
greenlet ------------------------------ 460.96 KiB/589.71 KiB
pyright ------------------------------ 158.16 KiB/5.31 MiB
ruff ------------------------------ 543.19 KiB/11.02 MiB
duckdb ------------------------------ 688.00 KiB/19.27 MiB
playwright ------------------------------ 64.00 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (5/33)
urllib3 ------------------------------ 125.66 KiB/125.66 KiB
werkzeug ------------------------------ 219.24 KiB/219.24 KiB
pytest ------------------------------ 203.88 KiB/335.58 KiB
greenlet ------------------------------ 476.96 KiB/589.71 KiB
pyright ------------------------------ 318.16 KiB/5.31 MiB
ruff ------------------------------ 671.19 KiB/11.02 MiB
duckdb ------------------------------ 832.00 KiB/19.27 MiB
playwright ------------------------------ 80.00 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (5/33)
urllib3 ------------------------------ 125.66 KiB/125.66 KiB
werkzeug ------------------------------ 219.24 KiB/219.24 KiB
pytest ------------------------------ 219.88 KiB/335.58 KiB
greenlet ------------------------------ 476.96 KiB/589.71 KiB
pyright ------------------------------ 350.16 KiB/5.31 MiB
ruff ------------------------------ 719.19 KiB/11.02 MiB
duckdb ------------------------------ 880.00 KiB/19.27 MiB
playwright ------------------------------ 96.00 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (5/33)
werkzeug ------------------------------ 219.24 KiB/219.24 KiB
pytest ------------------------------ 219.88 KiB/335.58 KiB
greenlet ------------------------------ 476.96 KiB/589.71 KiB
pyright ------------------------------ 430.16 KiB/5.31 MiB
ruff ------------------------------ 799.19 KiB/11.02 MiB
duckdb ------------------------------ 960.00 KiB/19.27 MiB
playwright ------------------------------ 96.00 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (5/33)
pytest ------------------------------ 219.88 KiB/335.58 KiB
greenlet ------------------------------ 476.96 KiB/589.71 KiB
pyright ------------------------------ 446.16 KiB/5.31 MiB
ruff ------------------------------ 815.19 KiB/11.02 MiB
duckdb ------------------------------ 976.00 KiB/19.27 MiB
playwright ------------------------------ 96.00 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (5/33)
pytest ------------------------------ 287.88 KiB/335.58 KiB
greenlet ------------------------------ 492.96 KiB/589.71 KiB
pyright ------------------------------ 910.16 KiB/5.31 MiB
ruff ------------------------------ 1.27 MiB/11.02 MiB
duckdb ------------------------------ 1.42 MiB/19.27 MiB
playwright ------------------------------ 176.00 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (5/33)
greenlet ------------------------------ 540.96 KiB/589.71 KiB
pyright ------------------------------ 1.33 MiB/5.31 MiB
ruff ------------------------------ 1.83 MiB/11.02 MiB
duckdb ------------------------------ 1.98 MiB/19.27 MiB
playwright ------------------------------ 685.47 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (5/33)
greenlet ------------------------------ 540.96 KiB/589.71 KiB
pyright ------------------------------ 1.37 MiB/5.31 MiB
ruff ------------------------------ 1.87 MiB/11.02 MiB
duckdb ------------------------------ 2.02 MiB/19.27 MiB
playwright ------------------------------ 749.47 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠸ Preparing packages... (27/33)
greenlet ------------------------------ 589.71 KiB/589.71 KiB
pyright ------------------------------ 1.78 MiB/5.31 MiB
ruff ------------------------------ 2.55 MiB/11.02 MiB
duckdb ------------------------------ 2.66 MiB/19.27 MiB
playwright ------------------------------ 1.37 MiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠸ Preparing packages... (27/33)
pyright ------------------------------ 1.80 MiB/5.31 MiB
ruff ------------------------------ 2.69 MiB/11.02 MiB
duckdb ------------------------------ 2.81 MiB/19.27 MiB
playwright ------------------------------ 1.51 MiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠸ Preparing packages... (27/33)
pyright ------------------------------ 1.81 MiB/5.31 MiB
ruff ------------------------------ 3.34 MiB/11.02 MiB
duckdb ------------------------------ 3.45 MiB/19.27 MiB
playwright ------------------------------ 2.14 MiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠸ Preparing packages... (27/33)
pyright ------------------------------ 1.84 MiB/5.31 MiB
ruff ------------------------------ 4.08 MiB/11.02 MiB
duckdb ------------------------------ 4.17 MiB/19.27 MiB
playwright ------------------------------ 2.83 MiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠸ Preparing packages... (27/33)
pyright ------------------------------ 1.89 MiB/5.31 MiB
ruff ------------------------------ 4.83 MiB/11.02 MiB
duckdb ------------------------------ 4.90 MiB/19.27 MiB
playwright ------------------------------ 3.58 MiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠼ Preparing packages... (28/33)
pyright ------------------------------ 1.92 MiB/5.31 MiB
ruff ------------------------------ 5.63 MiB/11.02 MiB
duckdb ------------------------------ 5.71 MiB/19.27 MiB
playwright ------------------------------ 4.39 MiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠼ Preparing packages... (28/33)
pyright ------------------------------ 1.97 MiB/5.31 MiB
ruff ------------------------------ 6.38 MiB/11.02 MiB
duckdb ------------------------------ 6.44 MiB/19.27 MiB
playwright ------------------------------ 5.14 MiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠼ Preparing packages... (28/33)
pyright ------------------------------ 2.00 MiB/5.31 MiB
ruff ------------------------------ 7.20 MiB/11.02 MiB
duckdb ------------------------------ 7.27 MiB/19.27 MiB
playwright ------------------------------ 5.97 MiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠼ Preparing packages... (28/33)
pyright ------------------------------ 2.03 MiB/5.31 MiB
ruff ------------------------------ 7.92 MiB/11.02 MiB
duckdb ------------------------------ 7.97 MiB/19.27 MiB
playwright ------------------------------ 6.70 MiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠴ Preparing packages... (28/33)
pyright ------------------------------ 2.08 MiB/5.31 MiB
ruff ------------------------------ 8.55 MiB/11.02 MiB
duckdb ------------------------------ 8.58 MiB/19.27 MiB
playwright ------------------------------ 7.31 MiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠴ Preparing packages... (28/33)
pyright ------------------------------ 2.11 MiB/5.31 MiB
ruff ------------------------------ 9.33 MiB/11.02 MiB
duckdb ------------------------------ 9.36 MiB/19.27 MiB
playwright ------------------------------ 8.11 MiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠴ Preparing packages... (28/33)
pyright ------------------------------ 2.12 MiB/5.31 MiB
ruff ------------------------------ 10.16 MiB/11.02 MiB
duckdb ------------------------------ 10.16 MiB/19.27 MiB
playwright ------------------------------ 8.91 MiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠴ Preparing packages... (28/33)
pyright ------------------------------ 2.14 MiB/5.31 MiB
ruff ------------------------------ 10.84 MiB/11.02 MiB
duckdb ------------------------------ 10.88 MiB/19.27 MiB
playwright ------------------------------ 9.61 MiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠦ Preparing packages... (28/33)
pyright ------------------------------ 2.16 MiB/5.31 MiB
duckdb ------------------------------ 11.10 MiB/19.27 MiB
playwright ------------------------------ 9.84 MiB/43.05 MiB
Built scubaduck @ file:///workspace/scubaduck
⠦ Preparing packages... (28/33)
pyright ------------------------------ 2.16 MiB/5.31 MiB
duckdb ------------------------------ 11.50 MiB/19.27 MiB
playwright ------------------------------ 10.25 MiB/43.05 MiB
⠦ Preparing packages... (28/33)
pyright ------------------------------ 2.17 MiB/5.31 MiB
duckdb ------------------------------ 11.73 MiB/19.27 MiB
playwright ------------------------------ 10.48 MiB/43.05 MiB
⠦ Preparing packages... (28/33)
pyright ------------------------------ 2.19 MiB/5.31 MiB
duckdb ------------------------------ 13.01 MiB/19.27 MiB
playwright ------------------------------ 11.76 MiB/43.05 MiB
⠦ Preparing packages... (28/33)
pyright ------------------------------ 2.20 MiB/5.31 MiB
duckdb ------------------------------ 14.95 MiB/19.27 MiB
playwright ------------------------------ 12.00 MiB/43.05 MiB
⠦ Preparing packages... (28/33)
pyright ------------------------------ 2.23 MiB/5.31 MiB
duckdb ------------------------------ 16.50 MiB/19.27 MiB
playwright ------------------------------ 12.83 MiB/43.05 MiB
⠧ Preparing packages... (30/33)
pyright ------------------------------ 2.25 MiB/5.31 MiB
duckdb ------------------------------ 17.75 MiB/19.27 MiB
playwright ------------------------------ 14.03 MiB/43.05 MiB
⠧ Preparing packages... (30/33)
pyright ------------------------------ 2.25 MiB/5.31 MiB
duckdb ------------------------------ 18.81 MiB/19.27 MiB
playwright ------------------------------ 15.06 MiB/43.05 MiB
⠧ Preparing packages... (30/33)
pyright ------------------------------ 2.31 MiB/5.31 MiB
duckdb ------------------------------ 19.26 MiB/19.27 MiB
playwright ------------------------------ 16.68 MiB/43.05 MiB
⠧ Preparing packages... (30/33)
pyright ------------------------------ 2.34 MiB/5.31 MiB
playwright ------------------------------ 17.16 MiB/43.05 MiB
⠧ Preparing packages... (30/33)
pyright ------------------------------ 2.39 MiB/5.31 MiB
playwright ------------------------------ 18.73 MiB/43.05 MiB
⠇ Preparing packages... (31/33)
pyright ------------------------------ 2.41 MiB/5.31 MiB
playwright ------------------------------ 20.81 MiB/43.05 MiB
⠇ Preparing packages... (31/33)
pyright ------------------------------ 2.44 MiB/5.31 MiB
playwright ------------------------------ 23.29 MiB/43.05 MiB
⠇ Preparing packages... (31/33)
pyright ------------------------------ 2.45 MiB/5.31 MiB
playwright ------------------------------ 25.68 MiB/43.05 MiB
⠇ Preparing packages... (31/33)
pyright ------------------------------ 2.51 MiB/5.31 MiB
playwright ------------------------------ 27.45 MiB/43.05 MiB
⠋ Preparing packages... (31/33)
pyright ------------------------------ 2.61 MiB/5.31 MiB
playwright ------------------------------ 29.26 MiB/43.05 MiB
⠋ Preparing packages... (31/33)
pyright ------------------------------ 2.65 MiB/5.31 MiB
playwright ------------------------------ 30.97 MiB/43.05 MiB
⠋ Preparing packages... (31/33)
pyright ------------------------------ 2.72 MiB/5.31 MiB
playwright ------------------------------ 32.81 MiB/43.05 MiB
⠋ Preparing packages... (31/33)
pyright ------------------------------ 2.76 MiB/5.31 MiB
playwright ------------------------------ 34.51 MiB/43.05 MiB
⠙ Preparing packages... (31/33)
pyright ------------------------------ 2.81 MiB/5.31 MiB
playwright ------------------------------ 36.33 MiB/43.05 MiB
⠙ Preparing packages... (31/33)
pyright ------------------------------ 2.84 MiB/5.31 MiB
playwright ------------------------------ 37.59 MiB/43.05 MiB
⠙ Preparing packages... (31/33)
pyright ------------------------------ 2.86 MiB/5.31 MiB
playwright ------------------------------ 38.95 MiB/43.05 MiB
⠙ Preparing packages... (31/33)
pyright ------------------------------ 2.87 MiB/5.31 MiB
playwright ------------------------------ 39.92 MiB/43.05 MiB
⠹ Preparing packages... (31/33)
pyright ------------------------------ 2.89 MiB/5.31 MiB
playwright ------------------------------ 40.89 MiB/43.05 MiB
⠹ Preparing packages... (31/33)
pyright ------------------------------ 2.98 MiB/5.31 MiB
playwright ------------------------------ 41.28 MiB/43.05 MiB
⠹ Preparing packages... (31/33)
pyright ------------------------------ 3.08 MiB/5.31 MiB
playwright ------------------------------ 41.82 MiB/43.05 MiB
⠹ Preparing packages... (31/33)
pyright ------------------------------ 3.11 MiB/5.31 MiB
⠹ Preparing packages... (31/33)
pyright ------------------------------ 3.12 MiB/5.31 MiB
⠸ Preparing packages... (32/33)
pyright ------------------------------ 3.22 MiB/5.31 MiB
⠸ Preparing packages... (32/33)
pyright ------------------------------ 3.27 MiB/5.31 MiB
⠸ Preparing packages... (32/33)
pyright ------------------------------ 3.33 MiB/5.31 MiB
⠼ Preparing packages... (32/33)
pyright ------------------------------ 3.37 MiB/5.31 MiB
⠼ Preparing packages... (32/33)
pyright ------------------------------ 3.50 MiB/5.31 MiB
⠼ Preparing packages... (32/33)
pyright ------------------------------ 3.59 MiB/5.31 MiB
⠼ Preparing packages... (32/33)
pyright ------------------------------ 3.75 MiB/5.31 MiB
⠼ Preparing packages... (32/33)
pyright ------------------------------ 3.84 MiB/5.31 MiB
⠴ Preparing packages... (32/33)
pyright ------------------------------ 4.06 MiB/5.31 MiB
⠴ Preparing packages... (32/33)
pyright ------------------------------ 4.19 MiB/5.31 MiB
⠴ Preparing packages... (32/33)
pyright ------------------------------ 4.31 MiB/5.31 MiB
⠴ Preparing packages... (32/33)
pyright ------------------------------ 4.42 MiB/5.31 MiB
⠦ Preparing packages... (32/33)
pyright ------------------------------ 4.51 MiB/5.31 MiB
⠦ Preparing packages... (32/33)
pyright ------------------------------ 4.79 MiB/5.31 MiB
⠦ Preparing packages... (32/33)
Prepared 33 packages in 2.91s
░░░░░░░░░░░░░░░░░░░░ [0/0] Installing wheels...
░░░░░░░░░░░░░░░░░░░░ [0/33] Installing wheels...
░░░░░░░░░░░░░░░░░░░░ [0/33] itsdangerous==2.2.0
░░░░░░░░░░░░░░░░░░░░ [1/33] itsdangerous==2.2.0
░░░░░░░░░░░░░░░░░░░░ [1/33] six==1.17.0
█░░░░░░░░░░░░░░░░░░░ [2/33] six==1.17.0
█░░░░░░░░░░░░░░░░░░░ [2/33] markupsafe==3.0.2
█░░░░░░░░░░░░░░░░░░░ [3/33] markupsafe==3.0.2
█░░░░░░░░░░░░░░░░░░░ [3/33] pytest-xdist==3.6.1
██░░░░░░░░░░░░░░░░░░ [4/33] pytest-xdist==3.6.1
██░░░░░░░░░░░░░░░░░░ [4/33] blinker==1.9.0
███░░░░░░░░░░░░░░░░░ [5/33] blinker==1.9.0
███░░░░░░░░░░░░░░░░░ [5/33] typing-extensions==4.13.2
███░░░░░░░░░░░░░░░░░ [6/33] typing-extensions==4.13.2
███░░░░░░░░░░░░░░░░░ [6/33] charset-normalizer==3.4.2
████░░░░░░░░░░░░░░░░ [7/33] charset-normalizer==3.4.2
████░░░░░░░░░░░░░░░░ [7/33] iniconfig==2.1.0
████░░░░░░░░░░░░░░░░ [8/33] iniconfig==2.1.0
████░░░░░░░░░░░░░░░░ [8/33] pyee==13.0.0
█████░░░░░░░░░░░░░░░ [9/33] pyee==13.0.0
███████████████████░ [32/33] pyright==1.1.400
████████████████████ [33/33] pyright==1.1.400
Installed 33 packages in 117ms
+ blinker==1.9.0
+ certifi==2025.4.26
+ charset-normalizer==3.4.2
+ click==8.2.0
+ duckdb==1.2.2
+ execnet==2.1.1
+ flask==3.1.1
+ greenlet==3.2.2
+ idna==3.10
+ iniconfig==2.1.0
+ itsdangerous==2.2.0
+ jinja2==3.1.6
+ markupsafe==3.0.2
+ nodeenv==1.9.1
+ packaging==25.0
+ playwright==1.52.0
+ pluggy==1.6.0
+ pyee==13.0.0
+ pyright==1.1.400
+ pytest==8.3.5
+ pytest-base-url==2.1.0
+ pytest-playwright==0.7.0
+ pytest-xdist==3.6.1
+ python-dateutil==2.9.0.post0
+ python-slugify==8.0.4
+ requests==2.32.3
+ ruff==0.11.10
+ scubaduck==0.1.0 (from file:///workspace/scubaduck)
+ six==1.17.0
+ text-unidecode==1.3
+ typing-extensions==4.13.2
+ urllib3==2.4.0
+ werkzeug==3.1.3
++ source .venv/bin/activate
+++ '[' -z '' ']'
+++ '[' -n x ']'
+++ SCRIPT_PATH=.venv/bin/activate
+++ '[' .venv/bin/activate = /tmp/rAOA2b-setup_script.sh ']'
+++ deactivate nondestructive
+++ unset -f pydoc
+++ '[' -z '' ']'
+++ '[' -z '' ']'
+++ hash -r
+++ '[' -z '' ']'
+++ unset VIRTUAL_ENV
+++ unset VIRTUAL_ENV_PROMPT
+++ '[' '!' nondestructive = nondestructive ']'
+++ VIRTUAL_ENV=/workspace/scubaduck/.venv
+++ '[' linux-gnu = cygwin ']'
+++ '[' linux-gnu = msys ']'
+++ export VIRTUAL_ENV
+++ '[' -z '' ']'
+++ unset SCRIPT_PATH
+++ _OLD_VIRTUAL_PATH=/root/.cargo/bin:/root/.rbenv/shims:/root/.rbenv/bin:/root/.rbenv/shims:/root/.local/share/swiftly/bin:/root/.bun/bin:/root/.nvm/versions/node/v22.15.1/bin:/root/.pyenv/shims:3441PYENV_ROOT/shims:/root/.pyenv/bin:/usr/local/go/bin:/root/go/bin:/root/.rbenv/bin:/root/.rbenv/shims:/root/.bun/bin:/root/.local/bin:/root/.pyenv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+++ PATH=/workspace/scubaduck/.venv/bin:/root/.cargo/bin:/root/.rbenv/shims:/root/.rbenv/bin:/root/.rbenv/shims:/root/.local/share/swiftly/bin:/root/.bun/bin:/root/.nvm/versions/node/v22.15.1/bin:/root/.pyenv/shims:3441PYENV_ROOT/shims:/root/.pyenv/bin:/usr/local/go/bin:/root/go/bin:/root/.rbenv/bin:/root/.rbenv/shims:/root/.bun/bin:/root/.local/bin:/root/.pyenv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+++ export PATH
+++ '[' xscubaduck '!=' x ']'
+++ VIRTUAL_ENV_PROMPT='(scubaduck) '
+++ export VIRTUAL_ENV_PROMPT
+++ '[' -z '' ']'
+++ '[' -z '' ']'
+++ _OLD_VIRTUAL_PS1=
+++ PS1='(scubaduck) '
+++ export PS1
+++ alias pydoc
+++ true
+++ hash -r
++ playwright install chromium
Downloading Chromium 136.0.7103.25 (playwright build v1169) from https://cdn.playwright.dev/dbazure/download/playwright/builds/chromium/1169/chromium-linux.zip
167.7 MiB [] 0% 0.0s167.7 MiB [] 0% 33.2s167.7 MiB [] 0% 30.9s167.7 MiB [] 0% 20.0s167.7 MiB [] 0% 12.6s167.7 MiB [] 0% 8.2s167.7 MiB [] 1% 6.4s167.7 MiB [] 1% 5.7s167.7 MiB [] 2% 5.2s167.7 MiB [] 3% 3.7s167.7 MiB [] 4% 3.1s167.7 MiB [] 5% 3.0s167.7 MiB [] 6% 3.0s167.7 MiB [] 6% 2.8s167.7 MiB [] 7% 2.7s167.7 MiB [] 9% 2.4s167.7 MiB [] 10% 2.3s167.7 MiB [] 11% 2.2s167.7 MiB [] 12% 2.0s167.7 MiB [] 13% 1.9s167.7 MiB [] 15% 1.8s167.7 MiB [] 16% 1.7s167.7 MiB [] 18% 1.6s167.7 MiB [] 20% 1.5s167.7 MiB [] 21% 1.5s167.7 MiB [] 22% 1.4s167.7 MiB [] 23% 1.4s167.7 MiB [] 24% 1.4s167.7 MiB [] 26% 1.3s167.7 MiB [] 27% 1.3s167.7 MiB [] 28% 1.3s167.7 MiB [] 30% 1.3s167.7 MiB [] 31% 1.2s167.7 MiB [] 32% 1.2s167.7 MiB [] 33% 1.2s167.7 MiB [] 34% 1.2s167.7 MiB [] 36% 1.1s167.7 MiB [] 37% 1.0s167.7 MiB [] 39% 1.0s167.7 MiB [] 40% 1.0s167.7 MiB [] 42% 0.9s167.7 MiB [] 43% 0.9s167.7 MiB [] 44% 0.9s167.7 MiB [] 46% 0.9s167.7 MiB [] 47% 0.8s167.7 MiB [] 49% 0.8s167.7 MiB [] 51% 0.8s167.7 MiB [] 52% 0.7s167.7 MiB [] 54% 0.7s167.7 MiB [] 55% 0.7s167.7 MiB [] 56% 0.7s167.7 MiB [] 57% 0.6s167.7 MiB [] 59% 0.6s167.7 MiB [] 61% 0.6s167.7 MiB [] 62% 0.5s167.7 MiB [] 63% 0.5s167.7 MiB [] 65% 0.5s167.7 MiB [] 66% 0.5s167.7 MiB [] 67% 0.5s167.7 MiB [] 69% 0.4s167.7 MiB [] 70% 0.4s167.7 MiB [] 72% 0.4s167.7 MiB [] 73% 0.4s167.7 MiB [] 74% 0.4s167.7 MiB [] 76% 0.3s167.7 MiB [] 77% 0.3s167.7 MiB [] 78% 0.3s167.7 MiB [] 80% 0.3s167.7 MiB [] 82% 0.2s167.7 MiB [] 83% 0.2s167.7 MiB [] 84% 0.2s167.7 MiB [] 85% 0.2s167.7 MiB [] 87% 0.2s167.7 MiB [] 88% 0.2s167.7 MiB [] 89% 0.1s167.7 MiB [] 91% 0.1s167.7 MiB [] 92% 0.1s167.7 MiB [] 94% 0.1s167.7 MiB [] 95% 0.1s167.7 MiB [] 97% 0.0s167.7 MiB [] 98% 0.0s167.7 MiB [] 100% 0.0s
Chromium 136.0.7103.25 (playwright build v1169) downloaded to /root/.cache/ms-playwright/chromium-1169
Downloading FFMPEG playwright build v1011 from https://cdn.playwright.dev/dbazure/download/playwright/builds/ffmpeg/1011/ffmpeg-linux.zip
2.3 MiB [] 0% 0.0s2.3 MiB [] 4% 0.4s2.3 MiB [] 10% 0.3s2.3 MiB [] 20% 0.2s2.3 MiB [] 45% 0.1s2.3 MiB [] 91% 0.0s2.3 MiB [] 100% 0.0s
FFMPEG playwright build v1011 downloaded to /root/.cache/ms-playwright/ffmpeg-1011
Downloading Chromium Headless Shell 136.0.7103.25 (playwright build v1169) from https://cdn.playwright.dev/dbazure/download/playwright/builds/chromium/1169/chromium-headless-shell-linux.zip
101.4 MiB [] 0% 0.0s101.4 MiB [] 0% 19.2s101.4 MiB [] 0% 18.2s101.4 MiB [] 0% 16.5s101.4 MiB [] 0% 6.9s101.4 MiB [] 2% 3.9s101.4 MiB [] 4% 2.4s101.4 MiB [] 4% 2.3s101.4 MiB [] 6% 1.9s101.4 MiB [] 8% 1.5s101.4 MiB [] 10% 1.5s101.4 MiB [] 10% 1.6s101.4 MiB [] 12% 1.5s101.4 MiB [] 14% 1.3s101.4 MiB [] 16% 1.2s101.4 MiB [] 18% 1.1s101.4 MiB [] 21% 0.9s101.4 MiB [] 24% 0.9s101.4 MiB [] 27% 0.8s101.4 MiB [] 29% 0.7s101.4 MiB [] 31% 0.7s101.4 MiB [] 33% 0.7s101.4 MiB [] 35% 0.7s101.4 MiB [] 38% 0.6s101.4 MiB [] 40% 0.6s101.4 MiB [] 43% 0.5s101.4 MiB [] 45% 0.5s101.4 MiB [] 49% 0.5s101.4 MiB [] 53% 0.4s101.4 MiB [] 57% 0.4s101.4 MiB [] 60% 0.3s101.4 MiB [] 64% 0.3s101.4 MiB [] 68% 0.2s101.4 MiB [] 72% 0.2s101.4 MiB [] 76% 0.2s101.4 MiB [] 80% 0.1s101.4 MiB [] 83% 0.1s101.4 MiB [] 86% 0.1s101.4 MiB [] 88% 0.1s101.4 MiB [] 90% 0.1s101.4 MiB [] 94% 0.0s101.4 MiB [] 98% 0.0s101.4 MiB [] 100% 0.0s
Chromium Headless Shell 136.0.7103.25 (playwright build v1169) downloaded to /root/.cache/ms-playwright/chromium_headless_shell-1169
++ echo 'source .venv/bin/activate'I'll take a look at the repository to get started.
bash: .venv/bin/activate: No such file or directory
root@60bb5aa46aa5:/workspace# ls
scubaduckroot@60bb5aa46aa5:/workspace# cd scubaduck
root@60bb5aa46aa5:/workspace/scubaduck# ls
AGENTS.md codex_setup.sh scubaduck tests
README.md pyproject.toml stubs uv.lockroot@60bb5aa46aa5:/workspace/scubaduck# cat AGENTS.md
- Use "source .venv/bin/activate" to activate the virtual environment before
running any commands
- ONLY use non-interactive commands like cat, sed, apply_patch to do edits.
Do NOT use interactive editors.
- Do NOT attempt to install packages. Only the packages specified in
pyproject.toml are available. You cannot add new packages. If you
desperately want another package, make a note of it in the final PR
description.
- Use conventional commits to format PR title
- There are no nested AGENTS.md files, this is the only agents file
- When using Playwright, ONLY use chromium browser. This browser is already
installed.
- Use "ruff check" to check lint, "ruff format" to autoformat files and
"pyright" to typecheck.
- When writing the PR description, include the original prompt supplied by the
user.
- When you add functionality to the server, add server tests to
tests/test_server.py. When you add features to the frontend, add frontend
tests to tests/test_web.pyI'll check the README and repo structure to understand the correct setup.
root@60bb5aa46aa5:/workspace/scubaduck# ls scubaduck
__init__.py sample.csv server.py staticroot@60bb5aa46aa5:/workspace/scubaduck# ls scubaduck/static
index.html jsroot@60bb5aa46aa5:/workspace/scubaduck# ls scubaduck/static/js
chip_input.js timeseries_chart.jsI'll take a look at the server code now.
root@60bb5aa46aa5:/workspace/scubaduck# sed -n '1,200p' scubaduck/server.py
from __future__ import annotations
from dataclasses import dataclass, field, replace
from typing import Any, Dict, List, Tuple
import re
from datetime import datetime, timedelta, timezone
import time
from pathlib import Path
import os
import sqlite3
import traceback
import duckdb
from dateutil import parser as dtparser
from flask import Flask, jsonify, request, send_from_directory
@dataclass
class Filter:
column: str
op: str
value: str | int | float | list[str] | None
@dataclass
class QueryParams:
start: str | None = None
end: str | None = None
order_by: str | None = None
order_dir: str = "ASC"
limit: int | None = None
columns: list[str] = field(default_factory=lambda: [])
filters: list[Filter] = field(default_factory=lambda: [])
derived_columns: dict[str, str] = field(default_factory=lambda: {})
graph_type: str = "samples"
group_by: list[str] = field(default_factory=lambda: [])
aggregate: str | None = None
show_hits: bool = False
x_axis: str | None = None
granularity: str = "Auto"
fill: str = "0"
def _load_database(path: Path) -> duckdb.DuckDBPyConnection:
ext = path.suffix.lower()
if ext == ".csv":
con = duckdb.connect()
con.execute(
f"CREATE TABLE events AS SELECT * FROM read_csv_auto('{path.as_posix
()}')"
)
elif ext in {".db", ".sqlite"}:
con = duckdb.connect()
sconn = sqlite3.connect(path)
info = sconn.execute("PRAGMA table_info(events)").fetchall()
col_defs = ", ".join(f"{r[1]} {r[2]}" for r in info)
con.execute(f"CREATE TABLE events ({col_defs})")
placeholders = ",".join("?" for _ in info)
for row in sconn.execute("SELECT * FROM events"):
con.execute(f"INSERT INTO events VALUES ({placeholders})", row)
sconn.close()
else:
con = duckdb.connect(path)
return con
_REL_RE = re.compile(
r"([+-]?\d+(?:\.\d*)?)\s*(hour|hours|day|days|week|weeks|fortnight|fortnight
s)",
re.IGNORECASE,
)
def parse_time(val: str | None) -> str | None:
"""Parse an absolute or relative time string into ``YYYY-MM-DD HH:MM:SS``.""
"
if val is None or val == "":
return None
s = val.strip()
if s.lower() == "now":
dt = datetime.now(timezone.utc)
return dt.replace(microsecond=0).strftime("%Y-%m-%d %H:%M:%S")
m = _REL_RE.fullmatch(s)
if m:
qty = float(m.group(1))
unit = m.group(2).lower()
delta: timedelta
if unit.startswith("hour"):
delta = timedelta(hours=qty)
elif unit.startswith("day"):
delta = timedelta(days=qty)
elif unit.startswith("week"):
delta = timedelta(weeks=qty)
elif unit.startswith("fortnight"):
delta = timedelta(weeks=2 * qty)
else: # pragma: no cover - defensive
raise ValueError(f"Unsupported unit: {unit}")
dt = datetime.now(timezone.utc) + delta
return dt.replace(microsecond=0).strftime("%Y-%m-%d %H:%M:%S")
dt = dtparser.parse(s)
return dt.replace(microsecond=0, tzinfo=None).strftime("%Y-%m-%d %H:%M:%S")
def _granularity_seconds(granularity: str, start: str | None, end: str | None) -
> int:
gran = granularity.lower()
mapping = {
"1 second": 1,
"5 seconds": 5,
"10 seconds": 10,
"30 seconds": 30,
"1 minute": 60,
"4 minutes": 240,
"5 minutes": 300,
"10 minutes": 600,
"15 minutes": 900,
"30 minutes": 1800,
"1 hour": 3600,
"3 hours": 10800,
"6 hours": 21600,
"1 day": 86400,
"1 week": 604800,
"30 days": 2592000,
}
if gran in mapping:
return mapping[gran]
if gran in {"auto", "fine"} and start and end:
try:
s = dtparser.parse(start)
e = dtparser.parse(end)
except Exception:
return 3600
total = max((e - s).total_seconds(), 1)
buckets = 100 if gran == "auto" else 500
return max(int(total // buckets), 1)
return 3600
def build_query(params: QueryParams, column_types: Dict[str, str] | None = None)
-> str:
select_parts: list[str] = []
group_cols = params.group_by[:]
if params.graph_type == "timeseries":
sec = _granularity_seconds(params.granularity, params.start, params.end)
x_axis = params.x_axis or "timestamp"
bucket_expr = (
f"TIMESTAMP 'epoch' + INTERVAL '{sec} second' * "
f"CAST(floor(epoch({x_axis})/{sec}) AS BIGINT)"
)
select_parts.append(f"{bucket_expr} AS bucket")
group_cols = ["bucket"] + group_cols
has_agg = bool(group_cols) or params.aggregate is not None
if has_agg:
select_cols = (
group_cols[1:] if params.graph_type == "timeseries" else group_cols
)
select_parts.extend(select_cols)
agg = (params.aggregate or "avg").lower()
def agg_expr(col: str) -> str:
if agg.startswith("p"):
quant = float(agg[1:]) / 100
return f"quantile({col}, {quant})"
if agg == "count distinct":
return f"count(DISTINCT {col})"
if agg == "avg" and column_types is not None:
ctype = column_types.get(col, "").upper()
if "TIMESTAMP" in ctype or "DATE" in ctype or "TIME" in ctype:
return (
"TIMESTAMP 'epoch' + INTERVAL '1 second' * "
f"CAST(avg(epoch({col})) AS BIGINT)"
)
return f"{agg}({col})"
for col in params.columns:
if col in group_cols:
continue
select_parts.append(f"{agg_expr(col)} AS {col}")
if params.show_hits:
select_parts.insert(len(group_cols), "count(*) AS Hits")
else:
select_parts.extend(params.columns)
if has_agg and params.derived_columns:
inner_params = replace(
params,
derived_columns={},
order_by=None,
limit=None,
)
inner_sql = build_query(inner_params, column_types)
outer_select = ["t.*"] + [
f"{expr} AS {name}" for name, expr in params.derived_columns.items()
]
query = f"SELECT {', '.join(outer_select)} FROM ({inner_sql}) t"
if params.order_by:
query += f" ORDER BY {params.order_by} {params.order_dir}"
elif params.graph_type == "timeseries":
query += " ORDER BY bucket"
if params.limit is not None:
query += f" LIMIT {params.limit}"root@60bb5aa46aa5:/workspace/scubaduck# sed -n '200,400p' scubaduck/server.py
query += f" LIMIT {params.limit}"
return query
for name, expr in params.derived_columns.items():
select_parts.append(f"{expr} AS {name}")
select_clause = ", ".join(select_parts) if select_parts else "*"
query = f"SELECT {select_clause} FROM events"
where_parts: list[str] = []
if params.start:
where_parts.append(f"timestamp >= '{params.start}'")
if params.end:
where_parts.append(f"timestamp <= '{params.end}'")
for f in params.filters:
op = f.op
if op in {"empty", "!empty"}:
val = "''"
else:
if f.value is None:
continue
if isinstance(f.value, list):
if not f.value:
continue
if op == "=":
vals = " OR ".join(f"{f.column} = '{v}'" for v in f.value)
where_parts.append(f"({vals})")
continue
val = f"'{f.value}'" if isinstance(f.value, str) else str(f.value)
if op == "contains":
where_parts.append(f"{f.column} ILIKE '%' || {val} || '%'")
elif op == "!contains":
where_parts.append(f"{f.column} NOT ILIKE '%' || {val} || '%'")
elif op == "empty":
where_parts.append(f"{f.column} = {val}")
elif op == "!empty":
where_parts.append(f"{f.column} != {val}")
else:
where_parts.append(f"{f.column} {op} {val}")
if where_parts:
query += " WHERE " + " AND ".join(where_parts)
if group_cols:
query += " GROUP BY " + ", ".join(group_cols)
if params.order_by:
query += f" ORDER BY {params.order_by} {params.order_dir}"
elif params.graph_type == "timeseries":
query += " ORDER BY bucket"
if params.limit is not None:
query += f" LIMIT {params.limit}"
return query
def create_app(db_file: str | Path | None = None) -> Flask:
app = Flask(__name__, static_folder="static")
if db_file is None:
env_db = os.environ.get("SCUBADUCK_DB")
if env_db:
db_file = env_db
db_path = Path(db_file or Path(__file__).with_name("sample.csv")).resolve()
con = _load_database(db_path)
column_types: Dict[str, str] = {
r[1]: r[2] for r in con.execute("PRAGMA table_info(events)").fetchall()
}
sample_cache: Dict[Tuple[str, str], Tuple[List[str], float]] = {}
CACHE_TTL = 60.0
CACHE_LIMIT = 200
@app.route("/")
def index() -> Any: # pyright: ignore[reportUnusedFunction]
assert app.static_folder is not None
return send_from_directory(app.static_folder, "index.html")
@app.route("/js/<path:filename>")
def js(filename: str) -> Any: # pyright: ignore[reportUnusedFunction]
assert app.static_folder is not None
folder = Path(app.static_folder) / "js"
return send_from_directory(folder, filename)
@app.route("/api/columns")
def columns() -> Any: # pyright: ignore[reportUnusedFunction]
rows = con.execute("PRAGMA table_info(events)").fetchall()
return jsonify([{"name": r[1], "type": r[2]} for r in rows])
def _cache_get(key: Tuple[str, str]) -> List[str] | None:
item = sample_cache.get(key)
if item is None:
return None
vals, ts = item
if time.time() - ts > CACHE_TTL:
del sample_cache[key]
return None
sample_cache[key] = (vals, time.time())
return vals
def _cache_set(key: Tuple[str, str], vals: List[str]) -> None:
sample_cache[key] = (vals, time.time())
if len(sample_cache) > CACHE_LIMIT:
oldest = min(sample_cache.items(), key=lambda kv: kv[1][1])[0]
del sample_cache[oldest]
@app.route("/api/samples")
def sample_values() -> Any: # pyright: ignore[reportUnusedFunction]
column = request.args.get("column")
substr = request.args.get("q", "")
if not column or column not in column_types:
return jsonify([])
ctype = column_types[column].upper()
if "CHAR" not in ctype and "STRING" not in ctype and "VARCHAR" not in ct
ype:
return jsonify([])
key = (column, substr)
cached = _cache_get(key)
if cached is not None:
return jsonify(cached)
rows = con.execute(
f"SELECT DISTINCT {column} FROM events WHERE CAST({column} AS VARCHA
R) ILIKE '%' || ? || '%' LIMIT 20",
[substr],
).fetchall()
values = [r[0] for r in rows]
_cache_set(key, values)
return jsonify(values)
@app.route("/api/query", methods=["POST"])
def query() -> Any: # pyright: ignore[reportUnusedFunction]
payload = request.get_json(force=True)
try:
start = parse_time(payload.get("start"))
end = parse_time(payload.get("end"))
except Exception as exc:
return jsonify({"error": str(exc)}), 400
params = QueryParams(
start=start,
end=end,
order_by=payload.get("order_by"),
order_dir=payload.get("order_dir", "ASC"),
limit=payload.get("limit"),
columns=payload.get("columns", []),
derived_columns=payload.get("derived_columns", {}),
graph_type=payload.get("graph_type", "samples"),
group_by=payload.get("group_by", []),
aggregate=payload.get("aggregate"),
show_hits=payload.get("show_hits", False),
x_axis=payload.get("x_axis"),
granularity=payload.get("granularity", "Auto"),
fill=payload.get("fill", "0"),
)
for f in payload.get("filters", []):
params.filters.append(Filter(f["column"], f["op"], f.get("value")))
if params.graph_type not in {"table", "timeseries"} and (
params.group_by or params.aggregate or params.show_hits
):
return (
jsonify(
{
"error": "group_by, aggregate and show_hits are only val
id for table or timeseries view"
}
),
400,
)
valid_cols = set(column_types.keys())
valid_cols.update(params.derived_columns.keys())
if params.graph_type == "timeseries":
if params.x_axis is None:
for cand in ["time", "timestamp"]:
if cand in valid_cols:
params.x_axis = cand
break
if params.x_axis is None or params.x_axis not in valid_cols:
return jsonify({"error": "Invalid x_axis"}), 400
ctype = column_types.get(params.x_axis, "").upper()
if not any(t in ctype for t in ["TIMESTAMP", "DATE", "TIME"]):
return jsonify({"error": "x_axis must be a time column"}), 400
for col in params.columns:
if col not in valid_cols:
return jsonify({"error": f"Unknown column: {col}"}), 400
for col in params.group_by:
if col not in valid_cols:
return jsonify({"error": f"Unknown column: {col}"}), 400
if params.order_by and params.order_by not in valid_cols:
return jsonify({"error": f"Unknown column: {params.order_by}"}), 400
if params.group_by or params.graph_type == "timeseries":
agg = (params.aggregate or "avg").lower()
if agg.startswith("p") or agg == "sum":
need_numeric = True
allow_time = False
elif agg == "avg" or agg in {"min", "max"}:
need_numeric = False
allow_time = True
else:
need_numeric = False
allow_time = False
if need_numeric or allow_time:
for c in params.columns:
if c in params.group_by or c == params.x_axis:
continue
if c not in column_types:
continue
ctype = column_types.get(c, "").upper()root@60bb5aa46aa5:/workspace/scubaduck# sed -n '400,800p' scubaduck/server.py
ctype = column_types.get(c, "").upper()
is_numeric = any(
t in ctype
for t in [
"INT",
"DECIMAL",
"REAL",
"DOUBLE",
"FLOAT",
"NUMERIC",
"HUGEINT",
]
)
is_time = "TIMESTAMP" in ctype or "DATE" in ctype or "TIME"
in ctype
if need_numeric and not is_numeric:
return (
jsonify(
{
"error": f"Aggregate {agg} cannot be applied
to column {c}",
}
),
400,
)
if allow_time and not (is_numeric or is_time):
return (
jsonify(
{
"error": f"Aggregate {agg} cannot be applied
to column {c}",
}
),
400,
)
sql = build_query(params, column_types)
try:
rows = con.execute(sql).fetchall()
except Exception as exc:
tb = traceback.format_exc()
print(f"Query failed:\n{sql}\n{tb}")
return (
jsonify({"sql": sql, "error": str(exc), "traceback": tb}),
400,
)
return jsonify({"sql": sql, "rows": rows})
return app
app = create_app()
if __name__ == "__main__":
app.run(debug=True)I need to check the tests to understand how to handle start/end defaults and time grouping.
root@60bb5aa46aa5:/workspace/scubaduck# ls tests
conftest.py test_server.py test_web.pyroot@60bb5aa46aa5:/workspace/scubaduck# sed -n '1,200p' tests/test_server.py
from __future__ import annotations
import json
from pathlib import Path
import duckdb
from scubaduck import server
import pytest
def test_basic_query() -> None:
app = server.app
client = app.test_client()
payload = {
"start": "2024-01-01 00:00:00",
"end": "2024-01-02 00:00:00",
"order_by": "timestamp",
"order_dir": "ASC",
"limit": 10,
"columns": ["timestamp", "event", "value", "user"],
"filters": [],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert data
rows = data["rows"]
# We expect first three rows (until 2024-01-02 00:00:00)
assert len(rows) == 3
assert rows[0][1] == "login"
assert rows[1][1] == "logout"
def test_js_served() -> None:
app = server.app
client = app.test_client()
rv = client.get("/js/chip_input.js")
assert rv.status_code == 200
assert b"initChipInput" in rv.data
def test_filter_multi_token() -> None:
app = server.app
client = app.test_client()
payload = {
"start": "2024-01-01 00:00:00",
"end": "2024-01-02 03:00:00",
"order_by": "timestamp",
"limit": 10,
"columns": ["timestamp", "event", "value", "user"],
"filters": [{"column": "user", "op": "=", "value": ["alice", "charlie"]}
],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert data
rows = data["rows"]
# Should only return rows for alice and charlie
assert len(rows) == 3
assert rows[0][3] == "alice"
assert rows[-1][3] == "charlie"
def test_empty_filter_is_noop() -> None:
app = server.app
client = app.test_client()
base_payload = {
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"limit": 100,
"columns": ["timestamp", "event", "value", "user"],
}
no_filter = {**base_payload, "filters": []}
empty_filter = {
**base_payload,
"filters": [{"column": "user", "op": "=", "value": None}],
}
rv1 = client.post(
"/api/query", data=json.dumps(no_filter), content_type="application/json
"
)
rv2 = client.post(
"/api/query", data=json.dumps(empty_filter), content_type="application/j
son"
)
rows1 = rv1.get_json()["rows"]
rows2 = rv2.get_json()["rows"]
assert rows1 == rows2
def test_select_columns() -> None:
app = server.app
client = app.test_client()
payload = {
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"order_by": "timestamp",
"limit": 10,
"columns": ["timestamp", "user"],
"filters": [],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert data
rows = data["rows"]
assert len(rows[0]) == 2
assert rows[0][1] == "alice"
def test_string_filter_ops() -> None:
app = server.app
client = app.test_client()
base = {
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"order_by": "timestamp",
"limit": 100,
"columns": ["timestamp", "event", "value", "user"],
}
contains = {
**base,
"filters": [{"column": "user", "op": "contains", "value": "ali"}],
}
rv = client.post(
"/api/query", data=json.dumps(contains), content_type="application/json"
)
rows = rv.get_json()["rows"]
assert all("ali" in r[3] for r in rows)
regex = {
**base,
"filters": [{"column": "user", "op": "~", "value": "^a.*"}],
}
rv = client.post(
"/api/query", data=json.dumps(regex), content_type="application/json"
)
rows = rv.get_json()["rows"]
assert all(r[3].startswith("a") for r in rows)
assert len(rows) == 2
not_empty = {**base, "filters": [{"column": "user", "op": "!empty"}]}
rv = client.post(
"/api/query", data=json.dumps(not_empty), content_type="application/json
"
)
assert len(rv.get_json()["rows"]) == 4
def _make_payload() -> dict[str, object]:
return {
"start": "2024-01-01 00:00:00",
"end": "2024-01-02 00:00:00",
"order_by": "timestamp",
"order_dir": "ASC",
"limit": 10,
"columns": ["timestamp", "event", "value", "user"],
"filters": [],
}
def test_database_types(tmp_path: Path) -> None:
csv_file = tmp_path / "events.csv"
csv_file.write_text(Path("scubaduck/sample.csv").read_text())
sqlite_file = tmp_path / "events.sqlite"
import sqlite3
conn = sqlite3.connect(sqlite_file)
conn.execute(
"CREATE TABLE events (timestamp TEXT, event TEXT, value INTEGER, user TE
XT)"
)
with open(csv_file) as f:
next(f)
for line in f:
ts, ev, val, user = line.strip().split(",")
conn.execute(
"INSERT INTO events VALUES (?, ?, ?, ?)", (ts, ev, int(val), use
r)
)
conn.commit()
conn.close() # pyright: ignore[reportUnknownMemberType, reportAttributeAcce
ssIssue]
duckdb_file = tmp_path / "events.duckdb"
con = duckdb.connect(duckdb_file)
con.execute(
f"CREATE TABLE events AS SELECT * FROM read_csv_auto('{csv_file.as_posix
()}')"
)
con.close() # pyright: ignore[reportUnknownMemberType, reportAttributeAcces
sIssue]
for db in (csv_file, sqlite_file, duckdb_file):
app = server.create_app(db)
client = app.test_client()
payload = _make_payload()
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/js
on"
)
rows = rv.get_json()["rows"]
assert len(rows) == 3root@60bb5aa46aa5:/workspace/scubaduck# sed -n '200,400p' tests/test_server.py
assert len(rows) == 3
def test_envvar_db(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
csv_file = tmp_path / "custom.csv"
csv_file.write_text("timestamp,event,value,user\n2024-01-01 00:00:00,login,5
,bob\n")
monkeypatch.setenv("SCUBADUCK_DB", str(csv_file))
app = server.create_app()
client = app.test_client()
payload = _make_payload()
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
rows = rv.get_json()["rows"]
assert len(rows) == 1
def test_group_by_table() -> None:
app = server.app
client = app.test_client()
payload = {
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "table",
"order_by": "user",
"limit": 10,
"columns": ["value"],
"group_by": ["user"],
"aggregate": "Sum",
"show_hits": True,
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
rows = rv.get_json()["rows"]
assert rows[0][0] == "alice"
assert rows[0][1] == 2
assert rows[0][2] == 40
def test_relative_time_query(monkeypatch: pytest.MonkeyPatch) -> None:
app = server.app
client = app.test_client()
from datetime import datetime
fixed_now = datetime(2024, 1, 2, 4, 0, 0)
class FixedDateTime(datetime):
@classmethod
def now(cls, tz=None): # type: ignore[override]
return fixed_now if tz is None else fixed_now.astimezone(tz)
monkeypatch.setattr(server, "datetime", FixedDateTime)
payload = {
"start": "-1 hour",
"end": "now",
"order_by": "timestamp",
"limit": 100,
"columns": ["timestamp", "event", "value", "user"],
"filters": [],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert len(data["rows"]) == 1
assert data["rows"][0][3] == "charlie"
def test_invalid_time_error() -> None:
app = server.app
client = app.test_client()
payload = {
"start": "nonsense",
"end": "now",
"order_by": "timestamp",
"limit": 10,
"columns": ["timestamp"],
"filters": [],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 400
assert "error" in data
def test_query_error_returns_sql_and_traceback() -> None:
app = server.app
client = app.test_client()
payload = {
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"columns": ["event"],
"group_by": ["user"],
"aggregate": "avg",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 400
assert "error" in data
def test_table_unknown_column_error() -> None:
app = server.app
client = app.test_client()
payload = {
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "table",
"order_by": "timestamp",
"limit": 100,
"columns": ["user", "Hits", "value"],
"group_by": ["user"],
"aggregate": "Count",
"show_hits": True,
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 400
assert "Unknown column" in data["error"]
def test_samples_view_rejects_group_by() -> None:
app = server.app
client = app.test_client()
payload = {
"graph_type": "samples",
"group_by": ["user"],
"columns": ["timestamp"],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 400
assert "only valid" in data["error"]
def test_table_avg_with_timestamp() -> None:
app = server.app
client = app.test_client()
payload = {
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "table",
"order_by": "user",
"limit": 100,
"columns": ["user", "timestamp", "value"],
"group_by": ["user"],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert "error" not in data
rows = data["rows"]
assert rows[0][0] == "alice"
from dateutil import parser
ts = parser.parse(rows[0][1]).replace(tzinfo=None)
assert ts == parser.parse("2024-01-01 12:00:00")
def test_timeseries_basic() -> None:
app = server.app
client = app.test_client()
payload = {
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "timeseries",
"limit": 100,
"group_by": ["user"],
"aggregate": "Count",
"columns": ["value"],
"x_axis": "timestamp",
"granularity": "1 day",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert len(data["rows"]) == 4
def test_timeseries_orders_by_xaxis() -> None:
app = server.app
client = app.test_client()
payload = {
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",root@60bb5aa46aa5:/workspace/scubaduck# sed -n '400,800p' tests/test_server.py
"end": "2024-01-03 00:00:00",
"graph_type": "timeseries",
"limit": 100,
"columns": ["value"],
"x_axis": "timestamp",
"granularity": "1 day",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
rows = data["rows"]
from dateutil import parser
timestamps = [parser.parse(r[0]).replace(tzinfo=None) for r in rows]
assert timestamps == sorted(timestamps)
def test_timeseries_string_column_error() -> None:
app = server.app
client = app.test_client()
payload = {
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "timeseries",
"limit": 7,
"columns": ["timestamp", "event", "value", "user"],
"x_axis": "timestamp",
"granularity": "1 hour",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 400
assert "Aggregate" in data["error"]
def test_derived_column_basic() -> None:
app = server.app
client = app.test_client()
payload = {
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"limit": 10,
"columns": ["timestamp"],
"derived_columns": {"val2": "value * 2"},
"filters": [],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert data["rows"][0][1] == 20
def test_timeseries_derived_column() -> None:
app = server.app
client = app.test_client()
payload = {
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "timeseries",
"granularity": "1 hour",
"limit": 7,
"columns": ["value"],
"derived_columns": {"derived_1": "value * 2"},
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
rows = data["rows"]
assert all(r[2] == r[1] * 2 for r in rows)root@60bb5aa46aa5:/workspace/scubaduck# sed -n '800,1200p' tests/test_server.pyI'll check out the tests in tests/test_web.py next.
)
def run_query(
page: Any,
url: str,
*,
start: str | None = None,
end: str | None = None,
order_by: str | None = None,
order_dir: str | None = "ASC",
limit: int | None = None,
group_by: list[str] | None = None,
aggregate: str | None = None,
) -> dict[str, Any]:
page.goto(url)
page.wait_for_selector("#order_by option", state="attached")
page.wait_for_selector("#order_dir", state="attached")
page.wait_for_function("window.lastResults !== undefined")
if start is not None:
page.fill("#start", start)
if end is not None:
page.fill("#end", end)
if order_by is not None:
select_value(page, "#order_by", order_by)
if order_dir is not None and order_dir == "DESC":
page.click("#order_dir")
if limit is not None:
page.fill("#limit", str(limit))
if group_by is not None:
select_value(page, "#graph_type", "table")
page.evaluate(
"g => { groupBy.chips = g; groupBy.renderChips(); }",
group_by,
)
if aggregate is not None:
select_value(page, "#graph_type", "table")
select_value(page, "#aggregate", aggregate)
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
return page.evaluate("window.lastResults")
def test_range_filters(page: Any, server_url: str) -> None:
data = run_query(
page,
server_url,
start="2024-01-02 00:00:00",
end="2024-01-02 04:00:00",
order_by="timestamp",
limit=100,
)
assert len(data["rows"]) == 2
from dateutil import parser
timestamps = [parser.parse(row[0]).replace(tzinfo=None) for row in data["row
s"]]
assert timestamps == [
parser.parse("2024-01-02 00:00:00"),
parser.parse("2024-01-02 03:00:00"),
]
def test_order_by(page: Any, server_url: str) -> None:
data = run_query(
page,
server_url,
start="2024-01-01 00:00:00",
end="2024-01-03 00:00:00",
order_by="value",
order_dir="DESC",
limit=100,
)
values = [row[2] for row in data["rows"]]
assert values == sorted(values, reverse=True)
def test_limit(page: Any, server_url: str) -> None:
data = run_query(
page,
server_url,
start="2024-01-01 00:00:00",
end="2024-01-03 00:00:00",
order_by="timestamp",
limit=2,
)
assert len(data["rows"]) == 2
def test_simple_filter(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
page.click("text=Add Filter")
filter_el = page.query_selector("#filters .filter:last-child")
assert filter_el
page.evaluate(
"arg => setSelectValue(arg.el.querySelector('.f-col'), arg.val)",
{"el": filter_el, "val": "user"},
)
val_input = filter_el.query_selector(".f-val")
val_input.click()
page.keyboard.type("alice")
page.keyboard.press("Enter")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
data = page.evaluate("window.lastResults")
assert len(data["rows"]) == 2
assert all(row[3] == "alice" for row in data["rows"])
def test_default_filter_and_layout(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
count = page.evaluate("document.querySelectorAll('#filters .filter').length"
)
assert count == 1
last_is_button = page.evaluate(
"document.querySelector('#filters').lastElementChild.id === 'add_filter'
"
)
assert last_is_button
position = page.evaluate(
"getComputedStyle(document.querySelector('#filters .filter button.remove
')).position"
)
assert position == "static"
def test_filter_remove_alignment(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
diff = page.evaluate(
"() => { const r=document.querySelector('#filters .filter-row').getBound
ingClientRect(); const x=document.querySelector('#filters .filter-row button.rem
ove').getBoundingClientRect(); return Math.abs(r.right - x.right); }"
)
assert diff <= 1
def test_header_and_tabs(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
header = page.text_content("#header")
assert "sample.csv" in header
assert "events" in header
assert page.is_visible("#settings")
assert page.is_hidden("#columns")
page.click("text=Columns")
assert page.is_visible("#columns")
cols = [c.strip() for c in page.locator("#column_groups li").all_inner_texts
()]
assert "timestamp" in cols
assert "event" in cols
page.click("text=View Settings")
assert page.is_visible("#settings")
btn_color = page.evaluate(
"getComputedStyle(document.querySelector('#dive')).backgroundColor"
)
assert "rgb(0, 128, 0)" == btn_color
sidebar_overflow = page.evaluate(
"getComputedStyle(document.querySelector('#sidebar')).overflowY"
)
view_overflow = page.evaluate(
"getComputedStyle(document.querySelector('#view')).overflowY"
)
assert sidebar_overflow == "auto"
assert view_overflow == "auto"
def test_graph_type_table_fields(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#graph_type", state="attached")
select_value(page, "#graph_type", "table")
assert page.is_visible("#group_by_field")
assert page.is_visible("#aggregate_field")
assert page.is_visible("#show_hits_field")
page.click("text=Columns")
assert not page.is_visible("text=Strings:")
def test_graph_type_timeseries_fields(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#graph_type", state="attached")
select_value(page, "#graph_type", "timeseries")
assert page.is_visible("#group_by_field")
assert page.is_visible("#aggregate_field")
assert page.is_visible("#x_axis_field")
assert page.is_visible("#granularity_field")
assert page.is_visible("#fill_field")
def test_timeseries_default_query(page: Any, server_url: str) -> None:
page.goto(server_url)assert not page.is_checked("#column_groups input[value='timestamp']")
def test_timeseries_single_bucket(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#graph_type", state="attached")
page.fill("#start", "2024-01-01 00:00:00")
page.fill("#end", "2024-01-01 00:00:00")
select_value(page, "#graph_type", "timeseries")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
path = page.get_attribute("#chart path", "d")
assert path is not None and "NaN" not in path
def test_help_and_alignment(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
titles = page.evaluate(
"Array.from(document.querySelectorAll('#settings .help')).map(e => e.tit
le)"
)
assert any("start/end of the time range" in t for t in titles)
text_align = page.evaluate(
"getComputedStyle(document.querySelector('#settings label')).textAlign"
)
assert text_align == "right"
def test_table_sorting(page: Any, server_url: str) -> None:
run_query(
page,
server_url,
start="2024-01-01 00:00:00",
end="2024-01-03 00:00:00",
order_by="timestamp",
order_dir="ASC",
limit=100,
)
# header alignment
align = page.evaluate(
"getComputedStyle(document.querySelector('#results th')).textAlign"
)
assert align == "right"
header = page.locator("#results th").nth(3)
def values() -> list[str]:
return page.locator("#results td:nth-child(4)").all_inner_texts()
orig_rows = values()
assert orig_rows == ["alice", "bob", "alice", "charlie"]
first_sql = page.evaluate("window.lastResults.sql")
header.click()
assert values() == sorted(orig_rows)
assert header.inner_text().endswith("▲")
color = page.evaluate(
"getComputedStyle(document.querySelector('#results th:nth-child(4)')).co
lor"
)
assert "0, 0, 255" in color
assert page.evaluate("window.lastResults.sql") == first_sql
header.click()
assert values() == sorted(orig_rows, reverse=True)
assert header.inner_text().endswith("▼")
header.click()
assert values() == orig_rows
assert header.inner_text() == "user"
color = page.evaluate(
"getComputedStyle(document.querySelector('#results th:nth-child(4)')).co
lor"
)
assert "0, 0, 255" not in color
def test_relative_dropdown(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
btn = page.query_selector('[data-target="start-select"]')
assert btn
btn.click()
page.click("#start-select div:text('-3 hours')")
assert page.input_value("#start") == "-3 hours"
def test_end_dropdown_now(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
page.click('[data-target="end-select"]')
page.click("#end-select div:text('now')")
assert page.input_value("#end") == "now"
def test_invalid_time_error_shown(page: Any, server_url: str) -> None:
data = run_query(
page,
server_url,
start="nonsense",
end="now",
order_by="timestamp",
)
assert "error" in data
msg = page.text_content("#view")
assert "nonsense" in msg
def test_table_avg_group_by(page: Any, server_url: str) -> None:
data = run_query(
page,
server_url,
start="2024-01-01 00:00:00",
end="2024-01-03 00:00:00",
order_by="timestamp",
group_by=["user"],
aggregate="Avg",
)
assert "error" not in data
assert len(data["rows"]) == 3
def test_column_toggle_and_selection(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
page.click("text=Columns")
page.wait_for_selector("#column_groups input", state="attached")
count = page.evaluate(
"document.querySelectorAll('#column_groups input:checked').length"
)
assert count == 4
page.click("#columns_none")
count = page.evaluate(
"document.querySelectorAll('#column_groups input:checked').length"
)
assert count == 0
page.click("#columns_all")
count = page.evaluate(
"document.querySelectorAll('#column_groups input:checked').length"
)
assert count == 4
page.uncheck("#column_groups input[value='value']")
page.click("text=View Settings")
page.fill("#start", "2024-01-01 00:00:00")
page.fill("#end", "2024-01-02 00:00:00")
select_value(page, "#order_by", "timestamp")
page.fill("#limit", "10")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
data = page.evaluate("window.lastResults")
assert len(data["rows"][0]) == 3
headers = page.locator("#results th").all_inner_texts()
assert "value" not in headers
def test_columns_links_alignment(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
page.click("text=Columns")
page.wait_for_selector("#column_groups input", state="attached")
tag = page.evaluate("document.getElementById('columns_all').tagName")
assert tag == "A"
align = page.evaluate(
"getComputedStyle(document.querySelector('#column_actions')).textAlign"
)
assert align == "right"
def test_column_group_links(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
page.click("text=Columns")
page.wait_for_selector("#column_groups a", state="attached")
tag = page.evaluate("document.querySelector('#column_groups .col-group a').t
agName")
assert tag == "A"
def test_column_group_links_float_right(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
page.click("text=Columns")
page.wait_for_selector("#column_groups .col-group .links", state="attached")
float_val = page.evaluate(
"getComputedStyle(document.querySelector('#column_groups .col-group .lin
ks')).float"
)
assert float_val == "right"page.wait_for_selector("text=alice")
page.keyboard.press("ArrowDown")
page.keyboard.press("Enter")
chips = page.evaluate(
"Array.from(document.querySelectorAll('#filters .filter:last-child .chip
')).map(c => c.firstChild.textContent)"
)
assert chips == ["ali"]
page.click("#filters .filter:last-child .chip .x")
page.wait_for_selector(".chip", state="detached")
def test_chip_copy_and_paste(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
page.evaluate(
"Object.defineProperty(navigator, 'clipboard', {value:{ _data: '', write
Text(t){ this._data = t; }, readText(){ return Promise.resolve(this._data); } }}
)"
)
page.click("text=Add Filter")
f = page.query_selector("#filters .filter:last-child")
assert f
page.evaluate(
"arg => setSelectValue(arg.el.querySelector('.f-col'), arg.val)",
{"el": f, "val": "user"},
)
inp = f.query_selector(".f-val")
inp.click()
page.keyboard.type("alice")
page.keyboard.press("Enter")
inp.click()
page.keyboard.type("bob")
page.keyboard.press("Enter")
f.query_selector(".chip-copy").click()
assert page.evaluate("navigator.clipboard._data") == "alice,bob"
page.evaluate(
"var f=document.querySelector('#filters .filter:last-child'); f.chips=[]
; f.querySelectorAll('.chip').forEach(c=>c.remove())"
)
page.wait_for_selector("#filters .chip", state="detached")
inp.click()
page.evaluate(
"var dt=new DataTransfer(); dt.setData('text/plain','alice,bob'); var e=
new ClipboardEvent('paste',{clipboardData:dt}); document.querySelector('#filters
.filter:last-child .f-val').dispatchEvent(e);"
)
chips = page.evaluate(
"Array.from(document.querySelectorAll('#filters .filter:last-child .chip
')).map(c => c.firstChild.textContent)"
)
assert chips[:2] == ["alice", "bob"]
page.evaluate(
"var f=document.querySelector('#filters .filter:last-child'); f.chips=[]
; f.querySelectorAll('.chip').forEach(c=>c.remove())"
)
page.wait_for_selector("#filters .chip", state="detached")
inp.click()
page.evaluate(
"var dt=new DataTransfer(); dt.setData('text/plain','alice,bob'); var e=
new ClipboardEvent('paste',{clipboardData:dt}); Object.defineProperty(e,'shiftKe
y',{value:true}); document.querySelector('#filters .filter:last-child .f-val').d
ispatchEvent(e);"
)
chips = page.evaluate(
"Array.from(document.querySelectorAll('#filters .filter:last-child .chip
')).map(c => c.firstChild.textContent)"
)
assert chips[-1] == "alice,bob"
def test_chip_dropdown_hides_on_outside_click(page: Any, server_url: str) -> Non
e:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
page.click("text=Add Filter")
f = page.query_selector("#filters .filter:last-child")
assert f
page.evaluate(
"arg => setSelectValue(arg.el.querySelector('.f-col'), arg.val)",
{"el": f, "val": "user"},
)
inp = f.query_selector(".f-val")
inp.click()
page.wait_for_selector("#filters .filter:last-child .chip-dropdown div")
page.click("#header")
page.wait_for_selector("#filters .filter:last-child .chip-dropdown", state="
hidden")
def test_chip_input_no_outline(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
page.click("text=Add Filter")
inp = page.query_selector("#filters .filter:last-child .f-val")
assert inp
inp.click()
outline = page.evaluate(
"getComputedStyle(document.querySelector('#filters .filter:last-child .f
-val')).outlineStyle"
)
assert outline == "none"
def test_chip_enter_keeps_focus(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
page.click("text=Add Filter")
f = page.query_selector("#filters .filter:last-child")
assert f
page.evaluate(
"arg => setSelectValue(arg.el.querySelector('.f-col'), arg.val)",
{"el": f, "val": "user"},
)
inp = f.query_selector(".f-val")
inp.click()
page.wait_for_selector("#filters .filter:last-child .chip-dropdown")
page.keyboard.type("alice")
page.keyboard.press("Enter")
focused = page.evaluate(
"document.activeElement === document.querySelector('#filters .filter:las
t-child .f-val')"
)
assert focused
...
)
assert color1 != color2
page.hover("#results tr:nth-child(2)")
hover_color = page.evaluate(
"getComputedStyle(document.querySelector('#results tr:nth-child(2) td'))
.backgroundColor"
)
assert hover_color != color1
page.click("#results tr:nth-child(2)")
selected_color = page.evaluate(
"getComputedStyle(document.querySelector('#results tr:nth-child(2) td'))
.backgroundColor"
)
assert "189, 228, 255" in selected_color
overflow = page.evaluate(
"var v=document.getElementById('view'); v.scrollWidth > v.clientWidth"
)
assert not overflow
def test_timestamp_rendering(page: Any, server_url: str) -> None:
run_query(
page,
server_url,
start="2024-01-01 00:00:00",
end="2024-01-02 00:00:00",
order_by="timestamp",
limit=1,
)
cell = page.text_content("#results td")
assert cell != "Invalid Date"
valid = page.evaluate("v => !isNaN(Date.parse(v))", cell)
assert valid
def test_url_query_persistence(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
page.wait_for_function("window.lastResults !== undefined")
page.fill("#start", "2024-01-01 00:00:00")
page.fill("#end", "2024-01-02 00:00:00")
page.fill("#limit", "1")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
first_url = page.url
first_rows = page.evaluate("window.lastResults.rows.length")
page.fill("#limit", "2")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
second_url = page.url
second_rows = page.evaluate("window.lastResults.rows.length")
assert second_rows != first_rows
assert first_url != second_url
page.go_back()
page.wait_for_function("window.lastResults !== undefined")
assert page.url == first_url
assert page.evaluate("window.lastResults.rows.length") == first_rows
def test_load_from_url(page: Any, server_url: str) -> None:
url = (
f"{server_url}?start=2024-01-01%2000:00:00&end=2024-01-02%2000:00:00"
"&order_by=timestamp&limit=2"
)
page.goto(url)
page.wait_for_selector("#order_by option", state="attached")
page.wait_for_function("window.lastResults !== undefined")
assert page.input_value("#start") == "2024-01-01 00:00:00"
assert page.input_value("#end") == "2024-01-02 00:00:00"
assert page.input_value("#limit") == "2"
assert page.evaluate("window.lastResults.rows.length") == 2
def test_empty_data_message(page: Any, server_url: str) -> None:
data = run_query(
page,
server_url,
start="2025-01-01 00:00:00",
end="2025-01-02 00:00:00",
order_by="timestamp",
limit=100,
)
assert data["rows"] == []
msg = page.text_content("#view")
assert "Empty data provided to table" in msg
def test_group_by_chip_from_url(page: Any, server_url: str) -> None:
url = f"{server_url}?graph_type=table&group_by=user&order_by=user&limit=10"
page.goto(url)
page.wait_for_selector("#group_by_field .chip", state="attached")
chips = page.evaluate(
"Array.from(document.querySelectorAll('#group_by_field .chip')).map(c =>
c.firstChild.textContent)"
)
assert chips == ["user"]
def test_group_by_autocomplete(page: Any, server_url: str) -> None:
page.goto(f"{server_url}?graph_type=table")
page.wait_for_selector("#group_by_field", state="visible")
inp = page.query_selector("#group_by_field .f-val")
assert inp
inp.click()
page.keyboard.type("us")
page.wait_for_selector("#group_by_field .chip-dropdown div")
options = page.locator("#group_by_field .chip-dropdown div").all_inner_texts
()
assert "user" in options
def test_group_by_copy_icon(page: Any, server_url: str) -> None:
page.goto(f"{server_url}?graph_type=table")
page.wait_for_selector("#group_by_field", state="visible")
icon = page.text_content("#group_by_field .chip-copy")
assert icon == "⎘"I’m continuing to scroll to capture more details I need.
)
assert border == "none"
def test_table_group_by_query(page: Any, server_url: str) -> None:
data = run_query(
page,
server_url,
start="2024-01-01 00:00:00",
end="2024-01-03 00:00:00",
order_by="timestamp",
limit=100,
group_by=["user"],
aggregate="Count",
)
assert "error" not in data
assert len(data["rows"]) == 3
def test_table_avg_no_group_by(page: Any, server_url: str) -> None:
data = run_query(
page,
server_url,
aggregate="Avg",
)
assert len(data["rows"]) == 1
row = data["rows"][0]
assert row[0] == 4
from dateutil import parser
ts = parser.parse(row[1]).replace(tzinfo=None)
assert ts == parser.parse("2024-01-01 13:00:00")
assert row[2] == 25
def test_table_headers_show_aggregate(page: Any, server_url: str) -> None:
run_query(
page,
server_url,
aggregate="Avg",
)
headers = page.locator("#results th").all_inner_texts()
assert "Hits" in headers
assert "timestamp (avg)" in headers
assert "value (avg)" in headers
def test_format_number_function(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
vals = page.evaluate(
"() => [formatNumber(815210), formatNumber(999.999), formatNumber(0.0004
), formatNumber(0)]"
)
assert vals == ["815.21 K", "999.999", "0.000", "0"]
def test_derived_column_query(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
page.click("text=Columns")
page.click("text=Add Derived")
expr = page.query_selector("#derived_list .derived textarea")
assert expr
expr.fill("value * 2")
page.click("text=View Settings")
page.fill("#start", "2024-01-01 00:00:00")
page.fill("#end", "2024-01-03 00:00:00")
page.fill("#limit", "10")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
data = page.evaluate("window.lastResults")
assert data["rows"][0][-1] == 20
def test_derived_column_remove(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
page.click("text=Columns")
page.click("text=Add Derived")
assert page.query_selector("#derived_list .derived button.remove")
page.click("#derived_list .derived button.remove")
count = page.evaluate("document.querySelectorAll('#derived_list .derived').l
ength")
assert count == 0
def test_sql_query_display(page: Any, server_url: str) -> None:
data = run_query(
page,
server_url,
start="2024-01-01 00:00:00",
end="2024-01-02 00:00:00",
order_by="timestamp",
limit=10,
)
sql = data["sql"]
displayed = page.text_content("#sql_query")
assert displayed is not None
assert displayed.strip() == sqlI’ll keep scrolling to check more tests related to timeseries.
root@60bb5aa46aa5:/workspace/scubaduck# sed -n '1600,2400p' tests/test_web.pyI'll move forward with implementing the graph-side functionality and fixing Start/End fields.
series_chart.js
function showTimeSeries(data) {
const view = document.getElementById('view');
if (data.rows.length === 0) {
view.innerHTML = '<p id="empty-message">Empty data provided to table</p>';
return;
}
const width = 600;
const height = 400;
view.innerHTML =
'<div id="legend"></div><svg id="chart" width="' +
width +
'" height="' +
height +
'"></svg>';
const svg = document.getElementById('chart');
const legend = document.getElementById('legend');
const groups = groupBy.chips || [];
const hasHits = document.getElementById('show_hits').checked ? 1 : 0;
const series = {};
let minX = Infinity,
maxX = -Infinity,
minY = Infinity,
maxY = -Infinity;
data.rows.forEach(r => {
const ts = new Date(r[0]).getTime();
const key = groups.map((_, i) => r[1 + i]).join(':') || 'all';
const val = Number(r[1 + groups.length + hasHits]);
if (!series[key]) series[key] = [];
series[key].push({ x: ts, y: val });
if (ts < minX) minX = ts;
if (ts > maxX) maxX = ts;
if (val < minY) minY = val;
if (val > maxY) maxY = val;
});
const colors = [
'#1f77b4',
'#ff7f0e',
'#2ca02c',
'#d62728',
'#9467bd',
'#8c564b',
'#e377c2',
];
let colorIndex = 0;
const xRange = maxX - minX || 1;
const yRange = maxY - minY || 1;
const xScale = x => ((x - minX) / xRange) * (width - 60) + 50;
const yScale = y => height - 30 - ((y - minY) / yRange) * (height - 60);
Object.keys(series).forEach(key => {
const pts = series[key];
const color = colors[colorIndex++ % colors.length];
const path = pts
.map((p, i) => (i === 0 ? 'M' : 'L') + xScale(p.x) + ' ' + yScale(p.y))
.join(' ');
const el = document.createElementNS('http://www.w3.org/2000/svg', 'path');
el.setAttribute('d', path);
el.setAttribute('fill', 'none');
el.setAttribute('stroke', color);
svg.appendChild(el);
const item = document.createElement('div');
item.textContent = key;
item.style.color = color;
legend.appendChild(item);
});
}I'll browse through the other JS files to gather more context.
root@60bb5aa46aa5:/workspace/scubaduck# sed -n '1,200p' scubaduck/static/js/chip
_input.js
function initChipInput(filter, fetchOptions) {
const input = filter.querySelector('.f-val');
const chipsEl = filter.querySelector('.chip-input');
const dropdown = filter.querySelector('.chip-dropdown');
const copyBtn = filter.querySelector('.chip-copy');
filter.chips = [];
filter.renderChips = renderChips;
filter.addChip = addChip;
let highlight = 0;
let dropdownLocked = false;
chipsEl.addEventListener('click', () => {
input.focus();
});
function renderChips() {
chipsEl.querySelectorAll('.chip').forEach(c => c.remove());
filter.chips.forEach((v, i) => {
const span = document.createElement('span');
span.className = 'chip';
span.textContent = v;
const x = document.createElement('span');
x.className = 'x';
x.textContent = '✖';
x.addEventListener('click', e => {
e.stopPropagation();
filter.chips.splice(i, 1);
renderChips();
input.focus();
});
span.appendChild(x);
chipsEl.insertBefore(span, input);
});
}
function hideDropdown() {
dropdown.style.display = 'none';
dropdownLocked = true;
}
function showDropdown() {
if (!dropdownLocked && document.activeElement === input) {
dropdown.style.display = 'block';
}
}
function updateHighlight() {
Array.from(dropdown.children).forEach((c, i) => {
c.classList.toggle('highlight', i === highlight);
});
}
function addChip(val) {
if (!val) return;
const i = filter.chips.indexOf(val);
if (i !== -1) {
filter.chips.splice(i, 1);
} else {
filter.chips.push(val);
}
input.value = '';
renderChips();
}
copyBtn.addEventListener('click', () => {
navigator.clipboard && navigator.clipboard.writeText(filter.chips.join(','))
;
});
input.addEventListener('paste', e => {
e.preventDefault();
const text = e.clipboardData.getData('text');
if (e.shiftKey) {
addChip(text.trim());
} else {
text.split(',').forEach(t => addChip(t.trim()));
}
hideDropdown();
});
input.addEventListener('keydown', e => {
if (e.key === 'ArrowDown') {
if (dropdown.style.display !== 'none') {
highlight = Math.min(highlight + 1, dropdown.children.length - 1);
updateHighlight();
}
e.preventDefault();
} else if (e.key === 'ArrowUp') {
if (dropdown.style.display !== 'none') {
highlight = Math.max(highlight - 1, 0);
updateHighlight();
}
e.preventDefault();
} else if (e.key === 'Backspace' && input.value === '') {
if (filter.chips.length > 0) {
filter.chips.pop();
renderChips();
}
} else if (e.key === 'Enter') {
if (dropdown.style.display !== 'none' && dropdown.children.length > 0) {
const val = dropdown.children[highlight].dataset.value;
if (val !== input.value.trim()) {
addChip(val);
} else {
addChip(input.value.trim());
}
} else {
addChip(input.value.trim());
}
hideDropdown();
e.preventDefault();
}
});
function renderDropdown(vals) {
dropdown.innerHTML = '';
const typed = input.value.trim();
if (typed) {
vals.splice(1, 0, typed);
}
vals.forEach((v, i) => {
const d = document.createElement('div');
d.textContent = v;
d.dataset.value = v;
d.addEventListener('mouseover', () => {
highlight = i;
updateHighlight();
});
d.addEventListener('mousedown', evt => {
evt.preventDefault();
addChip(v);
hideDropdown();
input.blur();
});
dropdown.appendChild(d);
});
if (vals.length) {
highlight = 0;
updateHighlight();
showDropdown();
} else {
hideDropdown();
}
}
function loadOptions() {
dropdownLocked = false;
if (!fetchOptions) {
dropdown.innerHTML = '';
return;
}
Promise.resolve(fetchOptions(input.value, filter)).then(values => {
renderDropdown(Array.isArray(values) ? values : []);
});
}
input.addEventListener('focus', loadOptions);
input.addEventListener('input', loadOptions);
document.addEventListener('click', evt => {
if (evt.target !== input) {
hideDropdown();
}
});
}I'll check out the index.html or related JS to see if it's controlling the query.
root@60bb5aa46aa5:/workspace/scubaduck# sed -n '1,120p' scubaduck/static/js
sed: read error on scubaduck/static/js: Is a directoryI'll just open up the index.html file now.
root@60bb5aa46aa5:/workspace/scubaduck# sed -n '1,200p' scubaduck/static/index.h
tml
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<title>ScubaDuck</title>
<style>
body { margin: 0; font-family: sans-serif; height: 100vh; display: flex; fle
x-direction: column; }
#header { padding: 10px; font-weight: bold; border-bottom: 1px solid #ccc; }
#content { flex: 1; display: flex; height: calc(100vh - 42px); overflow: hid
den; }
#sidebar { width: 450px; padding: 10px; border-right: 3px solid #ccc; overfl
ow-y: auto; display: flex; flex-direction: column; box-sizing: border-box; }
#sidebar-resizer { width: 5px; cursor: col-resize; background: #ccc; }
#view { flex: 1; padding: 10px; overflow-y: auto; overflow-x: auto; }
.field { display: flex; align-items: center; margin-bottom: 10px; }
.field label { width: 80px; text-align: right; margin-right: 5px; }
.help { margin-left: 4px; cursor: help; }
.rel-btn { margin-left: 4px; }
#tabs { display: flex; align-items: center; margin-bottom: 10px; }
#tabs .tab { margin-right: 5px; background: none; border: 1px solid #ccc; pa
dding: 4px 8px; cursor: pointer; width: 120px; text-align: center; box-sizing: b
order-box; }
#tabs .tab.active { background: #eee; font-weight: bold; }
#dive { margin-left: auto; background: green; color: white; border: none; pa
dding: 5px 10px; cursor: pointer; }
.tab-content { display: none; }
.tab-content.active { display: block; }
#filter_list { display: flex; flex-direction: column; }
#filters .filter {
border: 1px solid #ccc;
padding: 5px;
margin-bottom: 5px;
position: relative;
display: flex;
flex-direction: column;
}
#derived_columns .derived {
border: 1px solid #ccc;
padding: 5px;
margin-bottom: 5px;
display: flex;
flex-direction: column;
}
#derived_columns .derived-row {
display: flex;
margin-bottom: 5px;
}
#derived_columns .derived-row input[type="text"] {
margin-left: 5px;
flex: 1;
}
#derived_columns .derived-row button.remove {
margin-left: 5px;
width: 20px;
flex: 0 0 auto;
padding: 0;
text-align: center;
line-height: 1;
}
#derived_columns textarea {
width: 100%;
box-sizing: border-box;
}
#filters .filter-row { display: flex; margin-bottom: 5px; }
#filters .filter-row .f-col { flex: 1; }
#filters .filter-row .f-op {
margin-left: 5px;
width: fit-content;
flex: 0 0 auto;
}
.chip-input input {
border: none;
flex: 1;
min-width: 60px;
margin: 2px;
outline: none;
}
.chip-box { position: relative; }
.chip-input { display: flex; flex-wrap: wrap; border: 1px solid #ccc; paddin
g: 2px; min-height: 24px; }
.chip { background: #eee; border: 1px solid #999; padding: 2px 4px; margin:
2px; border-radius: 3px; display: flex; align-items: center; }
.chip .x { margin-left: 4px; cursor: pointer; }
.chip-copy { margin-left: 4px; cursor: pointer; background: none; border: no
ne; }
.chip-dropdown { position: absolute; left: 0; right: 0; top: 100%; backgroun
d: white; border: 1px solid #ccc; max-height: 120px; overflow-y: auto; z-index:
10; display: none; }
.chip-dropdown div { padding: 2px 4px; cursor: pointer; }
.chip-dropdown div.highlight { background: #bde4ff; }
.rel-box { position: relative; display: flex; }
.rel-dropdown { position: absolute; left: 0; right: 0; top: 100%; background
: white; border: 1px solid #ccc; z-index: 10; display: none; }
.rel-dropdown div { padding: 2px 4px; cursor: pointer; }
.rel-dropdown div:hover { background: #bde4ff; }
.dropdown { position: relative; display: inline-block; }
.dropdown-display {
border: 1px solid #ccc;
padding: 2px 18px 2px 4px;
cursor: pointer;
min-width: 80px;
position: relative;
}
.dropdown-display::after {
content: '\25BC';
position: absolute;
right: 4px;
pointer-events: none;
}
.dropdown-menu { position: absolute; left: 0; right: 0; top: 100%; backgroun
d: white; border: 1px solid #ccc; z-index: 10; max-height: 160px; overflow-y: au
to; display: none; }
.dropdown-menu input { width: 100%; box-sizing: border-box; padding: 2px 4px
; border: none; border-bottom: 1px solid #ccc; }
.dropdown-menu div { padding: 2px 4px; cursor: pointer; }
.dropdown-menu div.selected { background: #bde4ff; }
.dropdown-menu .option:hover { background: #eee; }
.dropdown-menu input::placeholder { color: #999; }
#filters .filter button.remove {
margin-left: 5px;
width: 20px;
flex: 0 0 auto;
padding: 0;
text-align: center;
line-height: 1;
}
#filters h4 { margin: 0 0 5px 0; }
table { border-collapse: collapse; min-width: 100%; }
th, td { border: 1px solid #ccc; padding: 4px; box-sizing: border-box; }
th { text-align: left; cursor: pointer; position: relative; }
th.sorted { color: blue; }
tr:nth-child(even) td { background: #f9f9f9; }
tr.selected td { background: #bde4ff !important; }
tr:hover:not(.selected) td { background: #eee; }
#column_actions {
text-align: right;
margin-bottom: 5px;
}
#column_actions a {
margin-left: 5px;
}
.col-group-header {
overflow: hidden;
}
.col-group-header .links {
float: right;
}
.col-group-header .links a {
margin-left: 5px;
}
/* Column resizer removed */
</style>
</head>
<body>
<div id="header">sample.csv - events <select id="graph_type"><option value="sa
mples">Samples</option><option value="table">Table</option><option value="timese
ries">Time Series</option></select></div>
<div id="content">
<div id="sidebar">
<div id="tabs">
<button class="tab active" data-tab="settings">View Settings</button>
<button class="tab" data-tab="columns">Columns</button>
<button id="dive" onclick="dive()">Dive</button>
</div>
<div id="settings" class="tab-content active">
<div class="field">
<label>Start<span class="help" title="Sets the start/end of the time r
ange to query. Can be any kind of datetime string. For example: 'April 23, 2014'
or 'yesterday'.">[?]</span></label>
<div class="rel-box">
<input id="start" type="text" />
<button type="button" class="rel-btn" data-target="start-select">	
660;</button>
<div id="start-select" class="rel-dropdown" data-input="start">
<div data-value="-1 hour">-1 hour</div>
<div data-value="-3 hours">-3 hours</div>
<div data-value="-12 hours">-12 hours</div>
<div data-value="-1 day">-1 day</div>
<div data-value="-3 days">-3 days</div>
<div data-value="-1 week">-1 week</div>
<div data-value="-1 fortnight">-1 fortnight</div>
<div data-value="-30 days">-30 days</div>
<div data-value="-90 days">-90 days</div>
</div>
</div>
</div>
<div class="field">
<label>End<span class="help" title="Sets the start/end of the time ran
ge to query. Can be any kind of datetime string. For example: 'April 23, 2014' o
r 'yesterday'.">[?]</span></label>
<div class="rel-box">
<input id="end" type="text" />
<button type="button" class="rel-btn" data-target="end-select">φ
0;</button>
<div id="end-select" class="rel-dropdown" data-input="end">
<div data-value="now">now</div>
<div data-value="-1 hour">-1 hour</div>
<div data-value="-3 hours">-3 hours</div>
<div data-value="-12 hours">-12 hours</div>
<div data-value="-1 day">-1 day</div>
<div data-value="-3 days">-3 days</div>
<div data-value="-1 week">-1 week</div>
<div data-value="-1 fortnight">-1 fortnight</div>
<div data-value="-30 days">-30 days</div>
<div data-value="-90 days">-90 days</div>
</div>
</div>
</div>
<div class="field">
<label>Order By<span class="help" title="Choose a column to sort resul
ts by.">[?]</span></label>
<select id="order_by"></select>
<button id="order_dir" type="button">ASC \u25B2</button>
</div>
<div class="field">
<label>Limit<span class="help" title="Choose the maximum number of res
ults to show in the chart after any aggregations have been applied. For example,
a limit of 10 will show no more than 10 rows for a table, etc.">[?]</span></lab
el>
<input id="limit" type="number" value="100" />
</div>
<div id="x_axis_field" class="field" style="display:none;">
<label>X-axis</label>
<select id="x_axis"></select>
</div>
<div id="granularity_field" class="field" style="display:none;">root@60bb5aa46aa5:/workspace/scubaduck# sed -n '200,400p' scubaduck/static/index
.html
<div id="granularity_field" class="field" style="display:none;">
<label>Granularity</label>
<select id="granularity">
<option>Auto</option>
<option>Fine</option>
<option>1 second</option>
<option>5 seconds</option>
<option>10 seconds</option>
<option>30 seconds</option>
<option>1 minute</option>
<option>4 minutes</option>
<option>5 minutes</option>
<option>10 minutes</option>
<option>15 minutes</option>
<option>30 minutes</option>
<option>1 hour</option>
<option>3 hours</option>
<option>6 hours</option>
<option>1 day</option>
<option>1 week</option>
<option>30 days</option>
</select>
</div>
<div id="fill_field" class="field" style="display:none;">
<label>Fill Missing Buckets</label>
<select id="fill">
<option value="0">Fill with 0 (Per Series)</option>
<option value="connect">Connect (Per Series)</option>
<option value="blank">Leave blank</option>
</select>
</div>
<div id="group_by_field" class="field" style="display:none;">
<label>Group By</label>
<div class="chip-box">
<div class="chip-input">
<input id="group_by" class="f-val" type="text">
<button type="button" class="chip-copy">⎘</button>
</div>
<div class="chip-dropdown"></div>
</div>
</div>
<div id="aggregate_field" class="field" style="display:none;">
<label>Aggregate</label>
<select id="aggregate">
<option>Avg</option>
<option>Count</option>
<option>Sum</option>
<option>Min</option>
<option>Max</option>
<option>Count Distinct</option>
<option>p5</option>
<option>p25</option>
<option>p50</option>
<option>p70</option>
<option>p75</option>
<option>p90</option>
<option>p95</option>
<option>p99</option>
<option>p99.9</option>
<option>p99.99</option>
</select>
</div>
<div id="show_hits_field" class="field" style="display:none;">
<label>Show Hits</label>
<input id="show_hits" type="checkbox" checked>
</div>
<div id="filters">
<h4>Filters<span class="help" title="You can create as many filters as
you want. You can either write a filter using a UI or manual SQL. In the UI, fi
lter consists of a column name, a relation (e.g., =, !=, <, >) and then a text f
ield. The text field is a token input. It accepts multiple tokens for = relation
, in which case we match using an OR for all options.">[?]</span></h4>
<div id="filter_list"></div>
<button id="add_filter" type="button" onclick="addFilter()">Add Filter
</button>
</div>
<div id="query_info" style="margin-top:10px;"></div>
</div>
<div id="columns" class="tab-content">
<div id="column_actions">
<a id="columns_all" href="#">All</a>
<a id="columns_none" href="#">None</a>
</div>
<div id="column_groups"></div>
<div id="derived_columns">
<h4>Derived Columns</h4>
<div id="derived_list"></div>
<button id="add_derived" type="button" onclick="addDerived()">Add Deri
ved</button>
</div>
</div>
</div>
<div id="sidebar-resizer"></div>
<div id="view">
<table id="results"></table>
</div>
</div>
<script src="/js/chip_input.js"></script>
<script src="/js/timeseries_chart.js"></script>
<script>
const allColumns = [];
const baseColumns = [];
const columnTypes = {};
const stringColumns = [];
const baseStringColumns = [];
const integerColumns = [];
const baseIntegerColumns = [];
const timeColumns = [];
const baseTimeColumns = [];
const derivedColumns = [];
let selectedColumns = [];
let displayType = 'samples';
let groupBy = {chips: [], addChip: () => {}, renderChips: () => {}};
function initDropdown(select) {
const wrapper = document.createElement('div');
wrapper.className = 'dropdown';
if (select.classList.contains('f-col')) {
wrapper.classList.add('f-col');
}
select.parentNode.insertBefore(wrapper, select);
wrapper.appendChild(select);
select.style.display = 'none';
const disp = document.createElement('div');
disp.className = 'dropdown-display';
function updateDisplay() {
const opt = select.options[select.selectedIndex];
disp.textContent = opt ? opt.textContent : '';
}
updateDisplay();
wrapper.appendChild(disp);
const menu = document.createElement('div');
menu.className = 'dropdown-menu';
const search = document.createElement('input');
search.placeholder = 'Search';
menu.appendChild(search);
const list = document.createElement('div');
menu.appendChild(list);
wrapper.appendChild(menu);
function close() {
menu.style.display = 'none';
}
function open() {
renderOptions();
menu.style.display = 'block';
search.focus();
}
disp.addEventListener('click', () => {
if (menu.style.display === 'block') {
close();
} else {
open();
}
});
document.addEventListener('click', e => {
if (!wrapper.contains(e.target)) {
close();
}
});
function renderOptions() {
const q = search.value.toLowerCase();
list.innerHTML = '';
Array.from(select.options).forEach(o => {
if (!o.textContent.toLowerCase().includes(q)) return;
const div = document.createElement('div');
div.className = 'option';
if (q) {
const text = o.textContent;
const idx = text.toLowerCase().indexOf(q);
if (idx !== -1) {
div.innerHTML =
text.slice(0, idx) +
'<u>' +
text.slice(idx, idx + q.length) +
'</u>' +
text.slice(idx + q.length);
} else {
div.textContent = text;
}
} else {
div.textContent = o.textContent;
}
if (o.value === select.value) div.classList.add('selected');
div.addEventListener('mousedown', evt => {
evt.preventDefault();
select.value = o.value;
select.dispatchEvent(new Event('change'));
updateDisplay();
close();
});
list.appendChild(div);
});
}
search.addEventListener('input', renderOptions);
select.addEventListener('change', updateDisplay);
}
// Sidebar resizing
const sidebar = document.getElementById('sidebar');
const sidebarResizer = document.getElementById('sidebar-resizer');
let sidebarWidth = parseInt(localStorage.getItem('sidebarWidth') || 450, 10);
sidebar.style.width = sidebarWidth + 'px';.html
sidebar.style.width = sidebarWidth + 'px';
let sidebarResize = false;
function startSidebarDrag(e) {
e.preventDefault();
sidebarResize = true;
document.addEventListener('mousemove', onSidebarDrag);
document.addEventListener('mouseup', stopSidebarDrag);
}
function onSidebarDrag(e) {
if (!sidebarResize) return;
sidebarWidth = Math.max(200, e.clientX - sidebar.getBoundingClientRect().left)
;
sidebar.style.width = sidebarWidth + 'px';
}
function stopSidebarDrag() {
document.removeEventListener('mousemove', onSidebarDrag);
document.removeEventListener('mouseup', stopSidebarDrag);
sidebarResize = false;
localStorage.setItem('sidebarWidth', sidebarWidth);
}
sidebarResizer.addEventListener('mousedown', startSidebarDrag);
let orderDir = 'ASC';
const orderDirBtn = document.getElementById('order_dir');
const graphTypeSel = document.getElementById('graph_type');
function updateOrderDirButton() {
orderDirBtn.textContent = orderDir + (orderDir === 'ASC' ? ' \u25B2' : ' \u25B
C');
}
function updateDisplayTypeUI() {
const showTable = graphTypeSel.value === 'table';
const showTS = graphTypeSel.value === 'timeseries';
document.getElementById('group_by_field').style.display = showTable || showTS
? 'flex' : 'none';
document.getElementById('aggregate_field').style.display = showTable || showTS
? 'flex' : 'none';
document.getElementById('show_hits_field').style.display = showTable ? 'flex'
: 'none';
document.getElementById('x_axis_field').style.display = showTS ? 'flex' : 'non
e';
document.getElementById('granularity_field').style.display = showTS ? 'flex' :
'none';
document.getElementById('fill_field').style.display = showTS ? 'flex' : 'none'
;
document.querySelectorAll('#column_groups .col-group').forEach(g => {
if (g.querySelector('.col-group-header').textContent.startsWith('Strings'))
{
g.style.display = showTable || showTS ? 'none' : '';
}
});
if (showTS) {
const lim = document.getElementById('limit');
if (!lim.dataset.setByUser) {
lim.value = '7';
}
document.querySelectorAll('#column_groups input').forEach(cb => {
if (isTimeColumn(cb.value) || isStringColumn(cb.value)) {
cb.checked = false;
}
});
document.getElementById('order_by').value = '';
updateSelectedColumns();
}
displayType = graphTypeSel.value;
}
orderDirBtn.addEventListener('click', () => {
orderDir = orderDir === 'ASC' ? 'DESC' : 'ASC';
updateOrderDirButton();
});
updateOrderDirButton();
graphTypeSel.addEventListener('change', updateDisplayTypeUI);
fetch('/api/columns').then(r => r.json()).then(cols => {
const orderSelect = document.getElementById('order_by');
const xAxisSelect = document.getElementById('x_axis');
const groupsEl = document.getElementById('column_groups');
const groups = {
time: {name: 'Time', cols: [], ul: null},
integer: {name: 'Integers', cols: [], ul: null},
string: {name: 'Strings', cols: [], ul: null}
};
cols.forEach(c => {
const t = c.type.toUpperCase();
columnTypes[c.name] = c.type;
allColumns.push(c.name);
baseColumns.push(c.name);
let g = 'string';
if (t.includes('INT')) g = 'integer';
if (t.includes('TIMESTAMP')) {
g = 'time';
timeColumns.push(c.name);
baseTimeColumns.push(c.name);
}
if (g === 'string') {
stringColumns.push(c.name);
baseStringColumns.push(c.name);
} else {
integerColumns.push(c.name);
baseIntegerColumns.push(c.name);
}
groups[g].cols.push(c.name);
if (g !== 'string') {
const o = document.createElement('option');
o.value = c.name;
o.textContent = c.name;
orderSelect.appendChild(o);
}
});
timeColumns.forEach(name => {
const o = document.createElement('option');
o.value = name;
o.textContent = name;
xAxisSelect.appendChild(o);
});
Object.keys(groups).forEach(key => {
const g = groups[key];
const div = document.createElement('div');
div.className = 'col-group';
const header = document.createElement('div');
header.className = 'col-group-header';
header.appendChild(document.createTextNode(g.name + ': '));
const links = document.createElement('span');
links.className = 'links';
const allBtn = document.createElement('a');
...
</div>
`;
const colSel = container.querySelector('.f-col');
colSel.innerHTML = allColumns.map(c => `<option value="${c}">${c}</option>`).j
oin('');
initDropdown(colSel);
function populateOps() {
const opSel = container.querySelector('.f-op');
const col = colSel.value;
const ops = isStringColumn(col)
? [
['=', '='],
['!=', '!='],
['~', 'matches regex'],
['!~', 'not matches regex'],
['contains', 'contains'],
['!contains', 'not contains'],
['empty', 'empty'],
['!empty', 'not empty'],
['LIKE', 'like'],
]
: [
['=', '='],
['!=', '!='],
['<', '<'],
['>', '>'],
];
opSel.innerHTML = ops.map(o => `<option value="${o[0]}">${o[1]}</option>`).j
oin('');
updateInputVis();
}
function updateInputVis() {
const op = container.querySelector('.f-op').value;
const box = container.querySelector('.chip-box');
box.style.display = op === 'empty' || op === '!empty' ? 'none' : 'block';
}
colSel.addEventListener('change', populateOps);
container.querySelector('.f-op').addEventListener('change', updateInputVis);
populateOps();
document.getElementById('filter_list').appendChild(container);
initChipInput(container, (typed, el) => {
const colEl = el.querySelector('.f-col select') || el.querySelector('.f-col'
);
if (!colEl) return [];
const col = colEl.value;
if (!isStringColumn(col)) return [];
return fetch(`/api/samples?column=${encodeURIComponent(col)}&q=${encodeURICo
mponent(typed)}`)
.then(r => r.json());
});
}
function nextDerivedName() {
let n = 1;
while (true) {
const name = `derived_${n}`;
if (!derivedColumns.some(d => d.name === name) && !allColumns.includes(name)
) return name;
n++;
}
}
function addDerived(data = {}) {
const container = document.createElement('div');
container.className = 'derived';
container.innerHTML = `
<div class="derived-row">
<select class="d-type">
<option value="aggregated">Aggregated</option>
<option value="string">String</option>
<option value="numeric">Numeric</option>
</select>
<input class="d-name" type="text">
<button type="button" class="remove" onclick="removeDerived(this)">✖</butt
on>
</div>
<label><input type="checkbox" class="d-use" checked> Include in Query</label
>
<textarea class="d-expr" rows="2"></textarea>
`;
document.getElementById('derived_list').appendChild(container);
const obj = {
type: data.type || 'string',
name: data.name || nextDerivedName(),
expr: data.expr || '',
include: data.include !== undefined ? data.include : true,
el: container
};
container.querySelector('.d-type').value = obj.type;
container.querySelector('.d-name').value = obj.name;
container.querySelector('.d-expr').value = obj.expr;
container.querySelector('.d-use').checked = obj.include;
['change','input'].forEach(evt => {
container.addEventListener(evt, refreshDerivedColumns);
});
derivedColumns.push(obj);
refreshDerivedColumns();
}
function removeDerived(btn) {
const el = btn.closest('.derived');
const idx = derivedColumns.findIndex(d => d.el === el);
if (idx !== -1) {
derivedColumns.splice(idx, 1);
}
el.remove();
refreshDerivedColumns();
}
function refreshDerivedColumns() {
allColumns.splice(0, allColumns.length, ...baseColumns);
stringColumns.splice(0, stringColumns.length, ...baseStringColumns);
integerColumns.splice(0, integerColumns.length, ...baseIntegerColumns);
timeColumns.splice(0, timeColumns.length, ...baseTimeColumns);
derivedColumns.forEach(d => {
d.type = d.el.querySelector('.d-type').value;
d.name = d.el.querySelector('.d-name').value;
d.expr = d.el.querySelector('.d-expr').value;
d.include = d.el.querySelector('.d-use').checked;
allColumns.push(d.name);
columnTypes[d.name] = d.type;Let me keep scrolling to the end to review the dive() function.
x.html
columnTypes[d.name] = d.type;
if (d.type === 'string') {
stringColumns.push(d.name);
} else {
integerColumns.push(d.name);
}
});
updateSelectedColumns();
}
let lastQueryTime = 0;
let queryStart = 0;
function dive(push=true) {
const params = collectParams();
if (push) {
history.pushState(params, '', paramsToSearch(params));
}
const payload = Object.assign({}, params);
const dcMap = {};
(params.derived_columns || []).forEach(d => {
if (d.include) dcMap[d.name] = d.expr;
});
payload.derived_columns = dcMap;
const view = document.getElementById('view');
view.innerHTML = '<p>Loading...</p>';
window.lastResults = undefined;
queryStart = performance.now();
fetch('/api/query', {method:'POST', headers:{'Content-Type':'application/json'
}, body:JSON.stringify(payload)})
.then(async r => {
const data = await r.json();
if (!r.ok) throw data;
return data;
})
.then(data => {
lastQueryTime = Math.round(performance.now() - queryStart);
showResults(data);
})
.catch(err => {
showError(err);
});
}
function collectParams() {
updateSelectedColumns();
const payload = {
start: document.getElementById('start').value,
end: document.getElementById('end').value,
order_by: document.getElementById('order_by').value,
order_dir: orderDir,
limit: parseInt(document.getElementById('limit').value, 10),
columns: selectedColumns.filter(c =>
c !== 'Hits' && !derivedColumns.some(dc => dc.name === c)
),
graph_type: graphTypeSel.value,
filters: Array.from(document.querySelectorAll('#filters .filter')).map(f =>
{
const chips = f.chips || [];
const op = f.querySelector('.f-op').value;
let value = null;
if (op !== 'empty' && op !== '!empty') {
value = chips.length === 0 ? null : (chips.length === 1 ? chips[0] : chi
ps);
}
const colSel = f.querySelector('.f-col select') || f.querySelector('.f-col
');
return {column: colSel.value, op, value};
}),
derived_columns: Array.from(document.querySelectorAll('#derived_list .derive
d')).map(d => ({
type: d.querySelector('.d-type').value,
name: d.querySelector('.d-name').value,
expr: d.querySelector('.d-expr').value,
include: d.querySelector('.d-use').checked,
}))
};
if (graphTypeSel.value === 'table' || graphTypeSel.value === 'timeseries') {
payload.group_by = groupBy.chips || [];
payload.aggregate = document.getElementById('aggregate').value;
payload.show_hits = document.getElementById('show_hits').checked;
}
if (graphTypeSel.value === 'timeseries') {
payload.x_axis = document.getElementById('x_axis').value;
payload.granularity = document.getElementById('granularity').value;
payload.fill = document.getElementById('fill').value;
}
return payload;
}
function paramsToSearch(params) {
const sp = new URLSearchParams();
if (params.start) sp.set('start', params.start);
if (params.end) sp.set('end', params.end);
if (params.order_by) sp.set('order_by', params.order_by);
if (params.order_dir) sp.set('order_dir', params.order_dir);
if (params.limit !== null && params.limit !== undefined) sp.set('limit', param
s.limit);
if (params.columns && params.columns.length) sp.set('columns', params.columns.
join(','));
if (params.filters && params.filters.length) sp.set('filters', JSON.stringify(
params.filters));
if (params.derived_columns && params.derived_columns.length) sp.set('derived_c
olumns', JSON.stringify(params.derived_columns));
if (params.graph_type) sp.set('graph_type', params.graph_type);
if (params.graph_type === 'table' || params.graph_type === 'timeseries') {
if (params.group_by && params.group_by.length) sp.set('group_by', params.gro
up_by.join(','));
if (params.aggregate) sp.set('aggregate', params.aggregate);
if (params.show_hits) sp.set('show_hits', '1');
}
if (params.graph_type === 'timeseries') {
if (params.x_axis) sp.set('x_axis', params.x_axis);
if (params.granularity) sp.set('granularity', params.granularity);
if (params.fill) sp.set('fill', params.fill);
}
const qs = sp.toString();
return qs ? '?' + qs : '';
}
function applyParams(params) {
document.getElementById('start').value = params.start || '';
document.getElementById('end').value = params.end || '';
...
});
row.forEach((v, i) => {
const col = selectedColumns[i];
const td = document.createElement('td');
if (isTimeColumn(col)) {
const d = new Date(v);
td.textContent = d.toLocaleString('en-US', {
weekday: 'short',
month: 'short',
day: 'numeric',
year: 'numeric',
hour: 'numeric',
minute: 'numeric',
second: 'numeric',
hour12: true,
timeZoneName: 'short'
});
} else {
if (col === 'Hits') {
const pct = totalHits ? ((v / totalHits) * 100).toFixed(1) : '0';
td.textContent = `${formatNumber(v)} (${pct}%)`;
} else {
td.textContent = isStringColumn(col) ? v : formatNumber(v);
}
}
td.style.textAlign = isStringColumn(col) ? 'left' : 'right';
tr.appendChild(td);
});
table.appendChild(tr);
});
// ensure table does not overflow unless necessary
const view = document.getElementById('view');
if (table.scrollWidth <= view.clientWidth) {
table.style.width = '100%';
}
}
function handleSort(e) {
const idx = parseInt(e.target.dataset.index, 10);
if (sortState.index !== idx) {
sortState.index = idx;
sortState.dir = 'asc';
} else if (sortState.dir === 'asc') {
sortState.dir = 'desc';
} else if (sortState.dir === 'desc') {
sortState.index = null;
sortState.dir = null;
} else {
sortState.dir = 'asc';
}
let rows = originalRows.slice();
if (sortState.index !== null) {
rows.sort((a, b) => {
const va = a[sortState.index];
const vb = b[sortState.index];
if (va === vb) return 0;
if (sortState.dir === 'desc') return va < vb ? 1 : -1;
return va > vb ? 1 : -1;
});
}
renderTable(rows);
}
function showResults(data) {
window.lastResults = data;
const view = document.getElementById('view');
if (graphTypeSel.value === 'timeseries') {
showTimeSeries(data);
} else {
if (data.rows.length === 0) {
view.innerHTML =
'<p id="empty-message">Empty data provided to table</p><table id="result
s"></table>';
} else {
view.innerHTML = '<table id="results"></table>';
}
originalRows = data.rows.slice();
sortState = {index: null, dir: null};
renderTable(originalRows);
}
const sqlEl = document.createElement('pre');
sqlEl.id = 'sql_query';
sqlEl.style.whiteSpace = 'pre-wrap';
sqlEl.style.marginTop = '10px';
sqlEl.textContent = data.sql;
view.appendChild(sqlEl);
document.getElementById('query_info').textContent =
`Your query took about ${lastQueryTime} ms`;
}
function showError(err) {
window.lastResults = err;
const view = document.getElementById('view');
let msg = '';
if (typeof err === 'string') {
msg = err;
} else if (err) {
msg = err.error || 'Error';
if (err.sql) {
msg += '\nSQL: ' + err.sql;
}
if (err.traceback) {
msg += '\n' + err.traceback;
}
}
view.innerHTML = `<pre id="error-message">${msg}</pre>`;
document.getElementById('query_info').textContent = '';
}
function setSelectValue(selector, value) {
const el = typeof selector === 'string' ? document.querySelector(selector) : s
elector;
if (el) {
const select = el.tagName === 'SELECT' ? el : el.querySelector('select');
if (select) {
select.value = value;
select.dispatchEvent(new Event('change'));
}
}
}
</script>
</body>
</html>I’m scrolling further to review additional functions after showError and others.
root@60bb5aa46aa5:/workspace/scubaduck# sed -n '1200,1600p' scubaduck/static/ind
ex.htmlI'll check out sample.csv to understand the dataset after reviewing the HTML and functions.
root@60bb5aa46aa5:/workspace/scubaduck# head -n 20 scubaduck/sample.csv
timestamp,event,value,user
2024-01-01 00:00:00,login,10,alice
2024-01-01 01:00:00,logout,20,bob
2024-01-02 00:00:00,login,30,alice
2024-01-02 03:00:00,login,40,charlieI'll take a look at the README to understand the timeseries dataset better.
root@60bb5aa46aa5:/workspace/scubaduck# sed -n '1,200p' README.md
I really like Scuba (Meta's internal real-time database system). The distributed
, real-time database part of Scuba is quite difficult (and expensive) to replica
te, but I also really like Scuba's UI for doing queries, and I have found myself
wishing that I have access to it even for "small" databases, e.g., I have a sql
ite dataset I want to explore.
Pivotal ideas:
* Time series by default. In the dedicated "time series" view, there are many fe
atures specifically oriented towards working towards tables that represent event
s that occurred over time: the start, end, compare, aggregate and granularity fi
elds all specially privilege the timestamp field. In fact, you can't log events
to Scuba's backing data store without a timestamp, they always come with one. (S
cuba also supports other views that don't presuppose a time series, but the time
series is the most beloved and well used view.) This is in contrast to typical
software which tries to generalize to arbitrary data first, with time series bei
ng added on later.
* It's all about exploration. Scuba is predicated on the idea that you don't kno
w what you're looking for, that you are going to spend time tweaking queries and
changing filters/grouping as part of an investigation to figure out why a syste
m behaves the way it is. So the filters/comparisons/groupings you want to edit a
re always visible on the left sidebar, with the expectation that you're going to
tweak the query to look at something else. Similarly, all the parameters of you
r query get saved into your URL, so your browser history can double up as a quer
y history / you can easily share a query with someone else. This is contrast to
typical software which is often oriented to making pretty dashboards and reports
. (This function is important too, but it's not what I want in exploration mode!
)
* You can fix data problems in the query editor. It's pretty common to have mess
ed up and ended up with a database that doesn't have exactly the columns you nee
d, or some columns that are corrupted in some way. Scuba has pretty robust suppo
rt for defining custom columns with arbitrary SQL functions, grouping over them
as if they were native functions, and doing so with minimal runtime cost (Scuba
aims to turn around your query in milliseconds!) Having to go and run a huge dat
a pipeline to fix your data is a big impediment to exploration; quick and easy c
ustom columns means you can patch over problems when you're investigating and fi
x them for real later.
We're going to build a exploratory data analysis tool like Scuba for time series
database (i.e., a database with a mandatory timestamp representing the time an
event occurred). We'll use DuckDB as the underlying SQL engine served from a Py
thon server, and render the GUI/results as a webpage with vanilla HTML and JS. W
e'll use choices.js to support token inputs. We define a token input to mean a
text input element where as you type a dropdown displays with valid values, and
if you select one or press enter, the selection turns into a token/chip that can
only be deleted as one unit.
To start, we are going to support one views: samples. The samples view only all
ows you to view individual samples from the database, subject to a filter. Our m
ain UI concept is that there is a left sidebar that is the query editor, and the
right side that shows the view. The sidebar is always visible and defaults to
the query parameters of the current view. After you make changes to the query,
clicking the "Dive" button updates the view. The URL of the page encodes all of
the values of the query (and gets updated when you Dive), so the browser's back
button lets you view previous queries.
The query editor's job is to generate a SQL query, which then is applied on the
database, and then the result visualized according to the view.
Here are the settings you can apply to the query. The help text should show up w
hen you mouse over the field name:
* Start/End - Help text: "Sets the start/end of the time range to query. Can be
any kind of datetime string. For example: 'April 23, 2014' or 'yesterday'." The
UI for this selector supports both relative selections (now, -1 hour, -3 hours,
-12 hours, -1 day, -3 days, -1 week, -1 fortnight, -30 days, -90 days) as well a
s specifying an absolute date. The way this field is rendered is there is a fre
e form text box, a drop down arrow (for the relative selectors), and then a cale
ndar button (for date selection).
* Order By - Help text: "Choose a column to sort results by." There is an ASC/D
ESC toggle next to it.
* Limit - Help text: "Choose the maximum number of results to show in the chart
after any aggregations have been applied. For example, a limit of 10 will show
no more than 10 rows for a table, etc."
* Filters - You can create as many filters as you want. You can either write a f
ilter using a UI or manual SQL. In the UI, filter consists of a column name, a r
elation (e.g., =, !=, <, >) and then a text field. The text field is a token inp
ut. It accepts multiple tokens for = relation, in which case we match using an O
R for all options.
There is also a "Columns" tab which lets you view all fields in the table, organ
ized by their type. You can also define derived columns, by specifying a column
name and SQL expression. Derived columns can be used for all parts of the UI, in
cluding filters/group by/etc. Columns have checkboxes indicating if we should SE
LECT them or not. Each selected column shows up in the graph. There is an All/N
one link which can be used to select/deselect all checkboxes.
The query UI constructs a SQL query that intuitively has this form:
```
SELECT column, column, ...,
FROM table
WHERE time >= min-timestamp
AND time <= max-timestamp
[AND condition ...]
ORDER BY aggregate(column)
LIMIT number
```
You should write tests for the server backend, demonstrating that at specific qu
ery values we get back the correct rows of data.
## Running the server
Activate the virtual environment and run the Flask development server:
```bash
flask --app scubaduck.server run --debug
```
By default the server loads `sample.csv`. Set the `SCUBADUCK_DB` environment
variable to point at a different database file (CSV, SQLite or DuckDB) if you
want to use another dataset.root@60bb5aa46aa5:/workspace/scubaduck# grep -n "rows" -n tests/test_web.py | he
ad
63: assert len(data["rows"]) == 2
66: timestamps = [parser.parse(row[0]).replace(tzinfo=None) for row in data["
rows"]]
83: values = [row[2] for row in data["rows"]]
96: assert len(data["rows"]) == 2
116: assert len(data["rows"]) == 2
117: assert all(row[3] == "alice" for row in data["rows"])
261: orig_rows = values()
262: assert orig_rows == ["alice", "bob", "alice", "charlie"]
267: assert values() == sorted(orig_rows)
276: assert values() == sorted(orig_rows, reverse=True)I need to check which tests call query() without start or end. Let's search.
root@60bb5aa46aa5:/workspace/scubaduck# grep -n "start" -n tests/test_server.py
| head
15: "start": "2024-01-01 00:00:00",
47: "start": "2024-01-01 00:00:00",
70: "start": "2024-01-01 00:00:00",
96: "start": "2024-01-01 00:00:00",
117: "start": "2024-01-01 00:00:00",
142: assert all(r[3].startswith("a") for r in rows)
154: "start": "2024-01-01 00:00:00",
221: "start": "2024-01-01 00:00:00",
256: "start": "-1 hour",
276: "start": "nonsense",I'm checking again in case there's a test without a starting point.
root@60bb5aa46aa5:/workspace/scubaduck# grep -n "start\":" -n tests/test_server.
py | grep -v "start": | grep -v "start\"" | headI'll look for 'dive(' or similar to find the server test cases.
root@60bb5aa46aa5:/workspace/scubaduck# grep -n "end": -n tests/test_server.py |
headI'm checking for existing tests on fill in tests/test_web.py, especially near the end.
root@60bb5aa46aa5:/workspace/scubaduck# grep -n "fill" -n tests/test_web.py
30: page.fill("#start", start)
32: page.fill("#end", end)
38: page.fill("#limit", str(limit))
196: assert page.is_visible("#fill_field")
216: page.fill("#start", "2024-01-01 00:00:00")
217: page.fill("#end", "2024-01-01 00:00:00")
357: page.fill("#start", "2024-01-01 00:00:00")
358: page.fill("#end", "2024-01-02 00:00:00")
360: page.fill("#limit", "10")
721: page.fill("#start", "2024-01-01 00:00:00")
722: page.fill("#end", "2024-01-02 00:00:00")
723: page.fill("#limit", "1")
730: page.fill("#limit", "2")
870: expr.fill("value * 2")
872: page.fill("#start", "2024-01-01 00:00:00")
873: page.fill("#end", "2024-01-03 00:00:00")
874: page.fill("#limit", "10")I need to add tests for fill, updating them due to the timeseries chart changes.
root@60bb5aa46aa5:/workspace/scubaduck# grep -n "chart" -n tests/test_web.py | h
ead
208: assert page.is_visible("#chart")
222: path = page.get_attribute("#chart path", "d")I'll take a look at the relevant lines around the timeseries default query.
page.goto(server_url)
page.wait_for_selector("#graph_type", state="attached")
select_value(page, "#graph_type", "table")
assert page.is_visible("#group_by_field")
assert page.is_visible("#aggregate_field")
assert page.is_visible("#show_hits_field")
page.click("text=Columns")
assert not page.is_visible("text=Strings:")
def test_graph_type_timeseries_fields(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#graph_type", state="attached")
select_value(page, "#graph_type", "timeseries")
assert page.is_visible("#group_by_field")
assert page.is_visible("#aggregate_field")
assert page.is_visible("#x_axis_field")
assert page.is_visible("#granularity_field")
assert page.is_visible("#fill_field")
def test_timeseries_default_query(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#graph_type", state="attached")
select_value(page, "#graph_type", "timeseries")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
data = page.evaluate("window.lastResults")
assert "error" not in data
assert page.is_visible("#chart")
page.click("text=Columns")
assert not page.is_checked("#column_groups input[value='timestamp']")
def test_timeseries_single_bucket(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#graph_type", state="attached")
page.fill("#start", "2024-01-01 00:00:00")
page.fill("#end", "2024-01-01 00:00:00")
select_value(page, "#graph_type", "timeseries")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
path = page.get_attribute("#chart path", "d")
assert path is not None and "NaN" not in path
def test_help_and_alignment(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
titles = page.evaluate(
"Array.from(document.querySelectorAll('#settings .help')).map(e => e.tit
le)"
)
assert any("start/end of the time range" in t for t in titles)
text_align = page.evaluate(
"getComputedStyle(document.querySelector('#settings label')).textAlign"
)
assert text_align == "right"
def test_table_sorting(page: Any, server_url: str) -> None:I’ll skip ahead 240 lines to look at more timeseries tests.
order_by="timestamp",
order_dir="ASC",
limit=100,
)
# header alignment
align = page.evaluate(
"getComputedStyle(document.querySelector('#results th')).textAlign"
)
assert align == "right"
header = page.locator("#results th").nth(3)
def values() -> list[str]:
return page.locator("#results td:nth-child(4)").all_inner_texts()
orig_rows = values()
assert orig_rows == ["alice", "bob", "alice", "charlie"]
first_sql = page.evaluate("window.lastResults.sql")
header.click()
assert values() == sorted(orig_rows)
assert header.inner_text().endswith("▲")
color = page.evaluate(
"getComputedStyle(document.querySelector('#results th:nth-child(4)')).co
lor"
)
assert "0, 0, 255" in color
assert page.evaluate("window.lastResults.sql") == first_sql
header.click()
assert values() == sorted(orig_rows, reverse=True)
assert header.inner_text().endswith("▼")
header.click()
assert values() == orig_rows
assert header.inner_text() == "user"
color = page.evaluate(
"getComputedStyle(document.querySelector('#results th:nth-child(4)')).co
lor"
)
assert "0, 0, 255" not in color
def test_relative_dropdown(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
btn = page.query_selector('[data-target="start-select"]')
assert btn
btn.click()
page.click("#start-select div:text('-3 hours')")
assert page.input_value("#start") == "-3 hours"
def test_end_dropdown_now(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
page.click('[data-target="end-select"]')
page.click("#end-select div:text('now')")
assert page.input_value("#end") == "now"
def test_invalid_time_error_shown(page: Any, server_url: str) -> None:
data = run_query(
page,
server_url,
start="nonsense",
end="now",
order_by="timestamp",
)
assert "error" in data
msg = page.text_content("#view")
assert "nonsense" in msg
def test_table_avg_group_by(page: Any, server_url: str) -> None:
data = run_query(
page,
server_url,
start="2024-01-01 00:00:00",
end="2024-01-03 00:00:00",
order_by="timestamp",
group_by=["user"],
aggregate="Avg",
)
assert "error" not in data
assert len(data["rows"]) == 3
def test_column_toggle_and_selection(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
page.click("text=Columns")
page.wait_for_selector("#column_groups input", state="attached")
count = page.evaluate(
"document.querySelectorAll('#column_groups input:checked').length"
)
assert count == 4
page.click("#columns_none")
count = page.evaluate(
"document.querySelectorAll('#column_groups input:checked').length"
)
assert count == 0
page.click("#columns_all")
count = page.evaluate(
"document.querySelectorAll('#column_groups input:checked').length"
)
assert count == 4
page.uncheck("#column_groups input[value='value']")
page.click("text=View Settings")
page.fill("#start", "2024-01-01 00:00:00")
page.fill("#end", "2024-01-02 00:00:00")
select_value(page, "#order_by", "timestamp")
page.fill("#limit", "10")I'll check around line 300 for tests related to timeseries features.
root@60bb5aa46aa5:/workspace/scubaduck# grep -n "timeseries" -n tests/test_web.p
y | tail
188:def test_graph_type_timeseries_fields(page: Any, server_url: str) -> None:
191: select_value(page, "#graph_type", "timeseries")
199:def test_timeseries_default_query(page: Any, server_url: str) -> None:
202: select_value(page, "#graph_type", "timeseries")
213:def test_timeseries_single_bucket(page: Any, server_url: str) -> None:
218: select_value(page, "#graph_type", "timeseries")I need tests to verify path drawing for missing buckets with different fill options. I'll focus on testing for '0', 'connect', and 'blank' fills using a dataset between 2024-01-01 00:00 and 03:00 with 1-hour granularity.
There’s no data at 02:00, so I'll confirm timeseries query results and check for gaps.
root@60bb5aa46aa5:/workspace/scubaduck# source .venv/bin/activate(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck# python - <<'PY'
> from scubaduck import server
> app = server.app
> client = app.test_client()
> payload = {
> "start":"2024-01-01 00:00:00",
> "end":"2024-01-02 03:00:00",
> "graph_type":"timeseries",
> "limit":100,
> "aggregate":"Count",
> "group_by":[],
> "columns":["value"],
> "x_axis":"timestamp",
> "granularity":"1 hour"
> }
> import json
> rv = client.post('/api/query',data=json.dumps(payload), content_type='applicat
ion/json')
> print(rv.get_json())
> PY{'rows': [['Mon, 01 Jan 2024 00:00:00 GMT', 1], ['Mon, 01 Jan 2024 01:00:00 GMT'
, 1], ['Tue, 02 Jan 2024 00:00:00 GMT', 1], ['Tue, 02 Jan 2024 03:00:00 GMT', 1]
], 'sql': "SELECT TIMESTAMP 'epoch' + INTERVAL '3600 second' * CAST(floor(epoch(
timestamp)/3600) AS BIGINT) AS bucket, count(value) AS value FROM events WHERE t
imestamp >= '2024-01-01 00:00:00' AND timestamp <= '2024-01-02 03:00:00' GROUP B
Y bucket ORDER BY bucket LIMIT 100"}
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck#There's a 23-hour gap between 01:00 and 1/2 00:00. I'll adjust the frontend.
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck# cat pyproject.toml
[project]
name = "scubaduck"
version = "0.1.0"
description = "Scuba on DuckDB"
readme = "README.md"
requires-python = ">=3.12"
license = {text = "MIT"}
dependencies = [
"duckdb",
"flask",
"python-dateutil>=2.9.0.post0",
]
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"
[tool.pytest.ini_options]
addopts = "-n auto --tb=native"
[tool.pyright]
# Pyright configuration with strict settings
include = ["scubaduck", "tests"]
exclude = ["**/__pycache__", "dist"]
stubPath = "stubs"
venvPath = "."
venv = ".venv"
reportMissingImports = true
reportMissingTypeStubs = true
pythonVersion = "3.12"
pythonPlatform = "All"
typeCheckingMode = "strict"
reportUnknownMemberType = true
reportUnknownParameterType = true
reportUnknownVariableType = true
reportUnknownArgumentType = true
reportPrivateImportUsage = true
reportUntypedFunctionDecorator = true
reportFunctionMemberAccess = true
reportIncompatibleMethodOverride = true
[tool.ruff]
target-version = "py312"
[tool.uv]
dev-dependencies = [
"pytest>=8.3.5",
"ruff>=0.11.2",
"pytest-xdist>=3.6.1",
"playwright>=1.52.0",
"pyright>=1.1.350",
"pytest-playwright>=0.7.0",
]
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck#I’ll update the server to calculate min and max timestamps when start/end aren’t provided.
The timestamp might return either a string or datetime object; I'll test to confirm.
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck# python - <<'PY'
> from scubaduck import server
> app=server.app
> con=server._load_database
> PY
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck#I'll explore the database to understand the return types for queries.
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck# python - <<'PY'
> import duckdb
> import pandas as pd
> import json
> con=duckdb.connect()
> con.execute("CREATE TABLE t (timestamp TIMESTAMP)")
> con.execute("INSERT INTO t VALUES ('2024-01-01 00:00:00')")
> print(con.execute("SELECT min(timestamp) FROM t").fetchall())
> print(type(con.execute("SELECT min(timestamp) FROM t").fetchone()[0]))
> PY
Traceback (most recent call last):
File "<stdin>", line 2, in <module>
ModuleNotFoundError: No module named 'pandas'(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck# python - <<'PY'
> import duckdb
> con=duckdb.connect()
> con.execute("CREATE TABLE t (timestamp TIMESTAMP)")
> con.execute("INSERT INTO t VALUES ('2024-01-01 00:00:00')")
> print(con.execute("SELECT min(timestamp) FROM t").fetchall())
> print(type(con.execute("SELECT min(timestamp) FROM t").fetchone()[0]))
> PY
[(datetime.datetime(2024, 1, 1, 0, 0),)]
<class 'datetime.datetime'>from __future__ import annotations
from typing import Any
def select_value(page: Any, selector: str, value: str) -> None:
page.evaluate(
"arg => setSelectValue(arg.sel, arg.val)",
{"sel": selector, "val": value},
)
def run_query(
page: Any,
url: str,
*,
start: str | None = None,
end: str | None = None,
order_by: str | None = None,
order_dir: str | None = "ASC",
limit: int | None = None,
group_by: list[str] | None = None,
aggregate: str | None = None,
) -> dict[str, Any]:
page.goto(url)
page.wait_for_selector("#order_by option", state="attached")
page.wait_for_selector("#order_dir", state="attached")
page.wait_for_function("window.lastResults !== undefined")
if start is not None:
page.fill("#start", start)
if end is not None:
page.fill("#end", end)
if order_by is not None:
select_value(page, "#order_by", order_by)
if order_dir is not None and order_dir == "DESC":
page.click("#order_dir")
if limit is not None:
page.fill("#limit", str(limit))
if group_by is not None:
select_value(page, "#graph_type", "table")
page.evaluate(
"g => { groupBy.chips = g; groupBy.renderChips(); }",
group_by,
)
if aggregate is not None:
select_value(page, "#graph_type", "table")
select_value(page, "#aggregate", aggregate)
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
return page.evaluate("window.lastResults")
def test_range_filters(page: Any, server_url: str) -> None:
data = run_query(
page,
server_url,
start="2024-01-02 00:00:00",
end="2024-01-02 04:00:00",
order_by="timestamp",
limit=100,
)
assert len(data["rows"]) == 2
from dateutil import parser
timestamps = [parser.parse(row[0]).replace(tzinfo=None) for row in data["row
s"]]
assert timestamps == [
parser.parse("2024-01-02 00:00:00"),
parser.parse("2024-01-02 03:00:00"),
]
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck#I'm thinking of implementing three separate timeseries tests with different fill options. However, I could consolidate them into one test for efficiency.
I'll implement a test for timeseries fill options and modify showTimeSeries for new parameters.
I'm using fill options to adjust series data parsing and visual scaling. Handling undefined values.
I’ll add a test to verify default start/end and bucket_size handling in timeseries queries.
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck# git status --short
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck#I'll open the server file near query() to make modifications.
if group_cols:
query += " GROUP BY " + ", ".join(group_cols)
if params.order_by:
query += f" ORDER BY {params.order_by} {params.order_dir}"
elif params.graph_type == "timeseries":
query += " ORDER BY bucket"
if params.limit is not None:
query += f" LIMIT {params.limit}"
return query
def create_app(db_file: str | Path | None = None) -> Flask:
app = Flask(__name__, static_folder="static")
if db_file is None:
env_db = os.environ.get("SCUBADUCK_DB")
if env_db:
db_file = env_db
db_path = Path(db_file or Path(__file__).with_name("sample.csv")).resolve()
con = _load_database(db_path)
column_types: Dict[str, str] = {
r[1]: r[2] for r in con.execute("PRAGMA table_info(events)").fetchall()
}
sample_cache: Dict[Tuple[str, str], Tuple[List[str], float]] = {}
CACHE_TTL = 60.0
CACHE_LIMIT = 200
@app.route("/")
def index() -> Any: # pyright: ignore[reportUnusedFunction]
assert app.static_folder is not None
return send_from_directory(app.static_folder, "index.html")
@app.route("/js/<path:filename>")
def js(filename: str) -> Any: # pyright: ignore[reportUnusedFunction]
assert app.static_folder is not None
folder = Path(app.static_folder) / "js"
return send_from_directory(folder, filename)
@app.route("/api/columns")
def columns() -> Any: # pyright: ignore[reportUnusedFunction]
rows = con.execute("PRAGMA table_info(events)").fetchall()
return jsonify([{"name": r[1], "type": r[2]} for r in rows])
def _cache_get(key: Tuple[str, str]) -> List[str] | None:
item = sample_cache.get(key)
if item is None:
return None
vals, ts = item
if time.time() - ts > CACHE_TTL:
del sample_cache[key]
return None
sample_cache[key] = (vals, time.time())
return vals
def _cache_set(key: Tuple[str, str], vals: List[str]) -> None:
sample_cache[key] = (vals, time.time())
if len(sample_cache) > CACHE_LIMIT:
oldest = min(sample_cache.items(), key=lambda kv: kv[1][1])[0]
del sample_cache[oldest]
@app.route("/api/samples")
def sample_values() -> Any: # pyright: ignore[reportUnusedFunction]
column = request.args.get("column")
substr = request.args.get("q", "")
if not column or column not in column_types:
return jsonify([])
ctype = column_types[column].upper()
if "CHAR" not in ctype and "STRING" not in ctype and "VARCHAR" not in ct
ype:
return jsonify([])
key = (column, substr)
cached = _cache_get(key)
if cached is not None:
return jsonify(cached)
rows = con.execute(
f"SELECT DISTINCT {column} FROM events WHERE CAST({column} AS VARCHA
R) ILIKE '%' || ? || '%' LIMIT 20",
[substr],
).fetchall()
values = [r[0] for r in rows]
_cache_set(key, values)
return jsonify(values)
@app.route("/api/query", methods=["POST"])
def query() -> Any: # pyright: ignore[reportUnusedFunction]
payload = request.get_json(force=True)
try:
start = parse_time(payload.get("start"))
end = parse_time(payload.get("end"))
except Exception as exc:
return jsonify({"error": str(exc)}), 400
params = QueryParams(
start=start,
end=end,
order_by=payload.get("order_by"),
order_dir=payload.get("order_dir", "ASC"),
limit=payload.get("limit"),
columns=payload.get("columns", []),
derived_columns=payload.get("derived_columns", {}),
graph_type=payload.get("graph_type", "samples"),
group_by=payload.get("group_by", []),
aggregate=payload.get("aggregate"),
show_hits=payload.get("show_hits", False),
x_axis=payload.get("x_axis"),
granularity=payload.get("granularity", "Auto"),
fill=payload.get("fill", "0"),
)
for f in payload.get("filters", []):
params.filters.append(Filter(f["column"], f["op"], f.get("value")))
if params.graph_type not in {"table", "timeseries"} and (
params.group_by or params.aggregate or params.show_hits
):
return (
jsonify(
{
"error": "group_by, aggregate and show_hits are only val
id for table or timeseries view"
}
),
400,
)
valid_cols = set(column_types.keys())
valid_cols.update(params.derived_columns.keys())
if params.graph_type == "timeseries":
if params.x_axis is None:
for cand in ["time", "timestamp"]:
if cand in valid_cols:
params.x_axis = cand
break
if params.x_axis is None or params.x_axis not in valid_cols:
return jsonify({"error": "Invalid x_axis"}), 400
ctype = column_types.get(params.x_axis, "").upper()
if not any(t in ctype for t in ["TIMESTAMP", "DATE", "TIME"]):
return jsonify({"error": "x_axis must be a time column"}), 400
for col in params.columns:
if col not in valid_cols:
return jsonify({"error": f"Unknown column: {col}"}), 400
for col in params.group_by:
if col not in valid_cols:
return jsonify({"error": f"Unknown column: {col}"}), 400
if params.order_by and params.order_by not in valid_cols:
return jsonify({"error": f"Unknown column: {params.order_by}"}), 400
if params.group_by or params.graph_type == "timeseries":
agg = (params.aggregate or "avg").lower()
if agg.startswith("p") or agg == "sum":
need_numeric = True
allow_time = False
elif agg == "avg" or agg in {"min", "max"}:
need_numeric = False
allow_time = True
else:
need_numeric = False
allow_time = False
if need_numeric or allow_time:
for c in params.columns:
if c in params.group_by or c == params.x_axis:
continue
if c not in column_types:
continue
ctype = column_types.get(c, "").upper()
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck#I’ll check the remaining part of the function towards the bottom.
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck# sed -n '400,800p' scubaduck/
server.py
ctype = column_types.get(c, "").upper()
is_numeric = any(
t in ctype
for t in [
"INT",
"DECIMAL",
"REAL",
"DOUBLE",
"FLOAT",
"NUMERIC",
"HUGEINT",
]
)
is_time = "TIMESTAMP" in ctype or "DATE" in ctype or "TIME"
in ctype
if need_numeric and not is_numeric:
return (
jsonify(
{
"error": f"Aggregate {agg} cannot be applied
to column {c}",
}
),
400,
)
if allow_time and not (is_numeric or is_time):
return (
jsonify(
{
"error": f"Aggregate {agg} cannot be applied
to column {c}",
}
),
400,
)
sql = build_query(params, column_types)
try:
rows = con.execute(sql).fetchall()
except Exception as exc:
tb = traceback.format_exc()
print(f"Query failed:\n{sql}\n{tb}")
return (
jsonify({"sql": sql, "error": str(exc), "traceback": tb}),
400,
)
return jsonify({"sql": sql, "rows": rows})
return app
app = create_app()
if __name__ == "__main__":
app.run(debug=True)
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck#I’m adding my changes near the build_query call. Time to patch now!
> *** Update File: scubaduck/server.py
> @@
> - sql = build_query(params, column_types)
> - try:
> - rows = con.execute(sql).fetchall()
> - except Exception as exc:
> - tb = traceback.format_exc()
> - print(f"Query failed:\n{sql}\n{tb}")
> - return (
> - jsonify({"sql": sql, "error": str(exc), "traceback": tb}),
> - 400,
> - )
> - return jsonify({"sql": sql, "rows": rows})
> + if params.start is None or params.end is None:
> + axis = params.x_axis or "timestamp"
> + mn, mx = con.execute(
> + f"SELECT min({axis}), max({axis}) FROM events"
> + ).fetchone()
> + if params.start is None and mn is not None:
> + params.start = (
> + mn.strftime("%Y-%m-%d %H:%M:%S") if not isinstance(mn, st
r) else mn
> + )
> + if params.end is None and mx is not None:
> + params.end = (
> + mx.strftime("%Y-%m-%d %H:%M:%S") if not isinstance(mx, st
r) else mx
> + )
> +
> + bucket_size: int | None = None
> + if params.graph_type == "timeseries":
> + bucket_size = _granularity_seconds(
> + params.granularity, params.start, params.end
> + )
> +
> + sql = build_query(params, column_types)
> + try:
> + rows = con.execute(sql).fetchall()
> + except Exception as exc:
> + tb = traceback.format_exc()
> + print(f"Query failed:\n{sql}\n{tb}")
> + return (
> + jsonify({"sql": sql, "error": str(exc), "traceback": tb}),
> + 400,
> + )
> +
> + result = {"sql": sql, "rows": rows}
> + if params.start is not None:
> + result["start"] = params.start
> + if params.end is not None:
> + result["end"] = params.end
> + if bucket_size is not None:
> + result["bucket_size"] = bucket_size
> + return jsonify(result)
> *** End Patch
> PATCH
Success. Updated the following files:
M scubaduck/server.py
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck#I’m updating the timeseries_chart.js now that the patch is applied.
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck# sed -n '1,160p' scubaduck/st
atic/js/timeseries_chart.js
function showTimeSeries(data) {
const view = document.getElementById('view');
if (data.rows.length === 0) {
view.innerHTML = '<p id="empty-message">Empty data provided to table</p>';
return;
}
const width = 600;
const height = 400;
view.innerHTML =
'<div id="legend"></div><svg id="chart" width="' +
width +
'" height="' +
height +
'"></svg>';
const svg = document.getElementById('chart');
const legend = document.getElementById('legend');
const groups = groupBy.chips || [];
const hasHits = document.getElementById('show_hits').checked ? 1 : 0;
const series = {};
let minX = Infinity,
maxX = -Infinity,
minY = Infinity,
maxY = -Infinity;
data.rows.forEach(r => {
const ts = new Date(r[0]).getTime();
const key = groups.map((_, i) => r[1 + i]).join(':') || 'all';
const val = Number(r[1 + groups.length + hasHits]);
if (!series[key]) series[key] = [];
series[key].push({ x: ts, y: val });
if (ts < minX) minX = ts;
if (ts > maxX) maxX = ts;
if (val < minY) minY = val;
if (val > maxY) maxY = val;
});
const colors = [
'#1f77b4',
'#ff7f0e',
'#2ca02c',
'#d62728',
'#9467bd',
'#8c564b',
'#e377c2',
];
let colorIndex = 0;
const xRange = maxX - minX || 1;
const yRange = maxY - minY || 1;
const xScale = x => ((x - minX) / xRange) * (width - 60) + 50;
const yScale = y => height - 30 - ((y - minY) / yRange) * (height - 60);
Object.keys(series).forEach(key => {
const pts = series[key];
const color = colors[colorIndex++ % colors.length];
const path = pts
.map((p, i) => (i === 0 ? 'M' : 'L') + xScale(p.x) + ' ' + yScale(p.y))
.join(' ');
const el = document.createElementNS('http://www.w3.org/2000/svg', 'path');
el.setAttribute('d', path);
el.setAttribute('fill', 'none');
el.setAttribute('stroke', color);
svg.appendChild(el);
const item = document.createElement('div');
item.textContent = key;
item.style.color = color;
legend.appendChild(item);
});
}
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck#I’ll apply a patch to this file and implement a new algorithm.
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck# apply_patch<<'PATCH'
> *** Begin Patch
> *** Update File: scubaduck/static/js/timeseries_chart.js
> @@
> -function showTimeSeries(data) {
> - const view = document.getElementById('view');
> - if (data.rows.length === 0) {
> - view.innerHTML = '<p id="empty-message">Empty data provided to table</p>'
;
> - return;
val;
+ });
+ });
+ if (fill === '0') {
+ if (minY > 0) minY = 0;
+ if (maxY < 0) maxY = 0;
+ }
+
+ const colors = > - }
> - const width = 600;
> - const height = 400;
> - view.innerHTML =
> - '<div id="legend"></div><svg id="chart" width="' +
> - width +
> - '" height="' +
> - height +
> - '"></svg>';
> - const svg = document.getElementById('chart');
> - const legend = document.getElementById('legend');
> - const groups = groupBy.chips || [];
> - const hasHits = document.getElementById('show_hits').checked ? 1 : 0;
> - const series = {};
> - let minX = Infinity,
> - maxX = -Infinity,
> - minY = Infinity,
> - maxY = -Infinity;
> - data.rows.forEach(r => {
> - const ts = new Date(r[0]).getTime();
> - const key = groups.map((_, i) => r[1 + i]).join(':') || 'all';
> - const val = Number(r[1 + groups.length + hasHits]);
> - if (!series[key]) series[key] = [];
> - series[key].push({ x: ts, y: val });
> - if (ts < minX) minX = ts;
> - if (ts > maxX) maxX = ts;
> - if (val < minY) minY = val;
> - if (val > maxY) maxY = val;
> - });
> - const colors = [
> - '#1f77b4',
> - '#ff7f0e',
> - '#2ca02c',
> - '#d62728',
> - '#9467bd',
> - '#8c564b',
> - '#e377c2',
> - ];
> - let colorIndex = 0;
> - const xRange = maxX - minX || 1;
> - const yRange = maxY - minY || 1;
> - const xScale = x => ((x - minX) / xRange) * (width - 60) + 50;
> - const yScale = y => height - 30 - ((y - minY) / yRange) * (height - 60);
> - Object.keys(series).forEach(key => {
> - const pts = series[key];
> - const color = colors[colorIndex++ % colors.length];
> - const path = pts
> - .map((p, i) => (i === 0 ? 'M' : 'L') + xScale(p.x) + ' ' + yScale(p.y))
> - .join(' ');
> - const el = document.createElementNS('http://www.w3.org/2000/svg', 'path')
;
> - el.setAttribute('d', path);
> - el.setAttribute('fill', 'none');
> - el.setAttribute('stroke', color);
> - svg.appendChild(el);
> - const item = document.createElement('div');
> - item.textContent = key;
> - item.style.color = color;
> - legend.appendChild(item);
> - });
> -}
> +function showTimeSeries(data) {
> + const view = document.getElementById('view');
> + if (data.rows.length === 0) {
> + view.innerHTML = '<p id="empty-message">Empty data provided to table</p>'
;
> + return;
> + }
> + const width = 600;
> + const height = 400;
> + view.innerHTML =
> + '<div id="legend"></div><svg id="chart" width="' +
> + width +
> + '" height="' +
> + height +
> + '"></svg>';
> + const svg = document.getElementById('chart');
> + const legend = document.getElementById('legend');
> + const groups = groupBy.chips || [];
> + const hasHits = document.getElementById('show_hits').checked ? 1 : 0;
> + const fill = document.getElementById('fill').value;
> + const bucketMs = (data.bucket_size || 3600) * 1000;
> + const start = data.start ? new Date(data.start).getTime() : null;
> + const end = data.end ? new Date(data.end).getTime() : null;
> + const series = {};
> + data.rows.forEach(r => {
> + const ts = new Date(r[0]).getTime();
> + const key = groups.map((_, i) => r[1 + i]).join(':') || 'all';
> + const val = Number(r[1 + groups.length + hasHits]);
> + if (!series[key]) series[key] = {};
> + series[key][ts] = val;
> + });
> +
> + const buckets = [];
> + let minX = start !== null ? start : Infinity;
> + let maxX = end !== null ? end : -Infinity;
> + if (start !== null && end !== null) {
> + for (let t = start; t <= end; t += bucketMs) {
> + buckets.push(t);
> + }
> + } else {
> + Object.keys(series).forEach(k => {
> + const s = series[k];
> + Object.keys(s).forEach(t => {
> + const n = Number(t);
> + if (n < minX) minX = n;
> + if (n > maxX) maxX = n;
> + });
> + });
> + for (let t = minX; t <= maxX; t += bucketMs) {
> + buckets.push(t);
> + }
> + }
> +
> + let minY = Infinity,
> + maxY = -Infinity;
> + Object.keys(series).forEach(key => {
> + const vals = series[key];
> + buckets.forEach(b => {
> + const v = vals[b];
> + const val = v === undefined && fill === '0' ? 0 : v;
> + if (val === undefined) return;
> + if (val < minY) minY = val;
> + if (val > maxY) maxY = val;
> + });
> + });
> + if (fill === '0') {
> + if (minY > 0) minY = 0;
> + if (maxY < 0) maxY = 0;
> + }
> +
> + const colors = [
> + '#1f77b4',
> + '#ff7f0e',
> + '#2ca02c',
> + '#d62728',
> + '#9467bd',
> + '#8c564b',
> + '#e377c2'
> + ];
> + let colorIndex = 0;
> + const xRange = maxX - minX || 1;
> + const yRange = maxY - minY || 1;
> + const xScale = x => ((x - minX) / xRange) * (width - 60) + 50;
> + const yScale = y => height - 30 - ((y - minY) / yRange) * (height - 60);
> +
> + Object.keys(series).forEach(key => {
> + const vals = series[key];
> + const color = colors[colorIndex++ % colors.length];
> + let path = '';
> + let drawing = false;
> + buckets.forEach(b => {
> + const v = vals[b];
> + if (v === undefined) {
> + if (fill === '0') {
> + const x = xScale(b);
> + const y = yScale(0);
> + path += (drawing ? 'L' : 'M') + x + ' ' + y + ' ';
> + drawing = true;
> + } else if (fill === 'blank') {
> + drawing = false;
> + }
> + // connect: do nothing
> + } else {
> + const x = xScale(b);
> + const y = yScale(v);
> + path += (drawing ? 'L' : 'M') + x + ' ' + y + ' ';
> + drawing = true;
> + }
> + });
> + const el = document.createElementNS('http://www.w3.org/2000/svg', 'path')
;
> + el.setAttribute('d', path.trim());
> + el.setAttribute('fill', 'none');
> + el.setAttribute('stroke', color);
> + svg.appendChild(el);
> + const item = document.createElement('div');
> + item.textContent = key;
> + item.style.color = color;
> + legend.appendChild(item);
> + });
> +}
> *** End Patch
> PATCH
Success. Updated the following files:
M scubaduck/static/js/timeseries_chart.js
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck#The patch seems to have replaced the whole file—checking for duplication or errors.
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck# sed -n '1,200p' scubaduck/st
atic/js/timeseries_chart.js
function showTimeSeries(data) {
const view = document.getElementById('view');
if (data.rows.length === 0) {
view.innerHTML = '<p id="empty-message">Empty data provided to table</p>';
return;
}
const width = 600;
const height = 400;
view.innerHTML =
'<div id="legend"></div><svg id="chart" width="' +
width +
'" height="' +
height +
'"></svg>';
const svg = document.getElementById('chart');
const legend = document.getElementById('legend');
const groups = groupBy.chips || [];
const hasHits = document.getElementById('show_hits').checked ? 1 : 0;
const fill = document.getElementById('fill').value;
const bucketMs = (data.bucket_size || 3600) * 1000;
const start = data.start ? new Date(data.start).getTime() : null;
const end = data.end ? new Date(data.end).getTime() : null;
const series = {};
data.rows.forEach(r => {
const ts = new Date(r[0]).getTime();
const key = groups.map((_, i) => r[1 + i]).join(':') || 'all';
const val = Number(r[1 + groups.length + hasHits]);
if (!series[key]) series[key] = {};
series[key][ts] = val;
});
const buckets = [];
let minX = start !== null ? start : Infinity;
let maxX = end !== null ? end : -Infinity;
if (start !== null && end !== null) {
for (let t = start; t <= end; t += bucketMs) {
buckets.push(t);
}
} else {
Object.keys(series).forEach(k => {
const s = series[k];
Object.keys(s).forEach(t => {
const n = Number(t);
if (n < minX) minX = n;
if (n > maxX) maxX = n;
});
});
for (let t = minX; t <= maxX; t += bucketMs) {
buckets.push(t);
}
}
let minY = Infinity,
maxY = -Infinity;
Object.keys(series).forEach(key => {
const vals = series[key];
buckets.forEach(b => {
const v = vals[b];
const val = v === undefined && fill === '0' ? 0 : v;
if (val === undefined) return;
if (val < minY) minY = val;
if (val > maxY) maxY = val;
});
});
if (fill === '0') {
if (minY > 0) minY = 0;
if (maxY < 0) maxY = 0;
}
const colors = [
'#1f77b4',
'#ff7f0e',
'#2ca02c',
'#d62728',
'#9467bd',
'#8c564b',
'#e377c2'
];
let colorIndex = 0;
const xRange = maxX - minX || 1;
const yRange = maxY - minY || 1;
const xScale = x => ((x - minX) / xRange) * (width - 60) + 50;
const yScale = y => height - 30 - ((y - minY) / yRange) * (height - 60);
Object.keys(series).forEach(key => {
const vals = series[key];
const color = colors[colorIndex++ % colors.length];
let path = '';
let drawing = false;
buckets.forEach(b => {
const v = vals[b];
if (v === undefined) {
if (fill === '0') {
const x = xScale(b);
const y = yScale(0);
path += (drawing ? 'L' : 'M') + x + ' ' + y + ' ';
drawing = true;
} else if (fill === 'blank') {
drawing = false;
}
// connect: do nothing
} else {
const x = xScale(b);
const y = yScale(v);
path += (drawing ? 'L' : 'M') + x + ' ' + y + ' ';
drawing = true;
}
});
const el = document.createElementNS('http://www.w3.org/2000/svg', 'path');
el.setAttribute('d', path.trim());
el.setAttribute('fill', 'none');
el.setAttribute('stroke', color);
svg.appendChild(el);
const item = document.createElement('div');
item.textContent = key;
item.style.color = color;
legend.appendChild(item);
});
}
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck#I'll go ahead and run ruff format, ruff check, pyright, and pytest in sequence.
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck# ruff format scubaduck/server
.py scubaduck/static/js/timeseries_chart.js >/tmp/format.log && tail -n 20 /tmp/
format.log
error: Failed to parse scubaduck/static/js/timeseries_chart.js:1:10: Simple stat
ements must be separated by newlines or semicolons
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck#I misapplied ruff format—it's for Python. I'll only run it on server.py.
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck# ruff format scubaduck/server
.py >/tmp/format.log && tail -n 20 /tmp/format.log
1 file left unchanged(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck# ruff check >/tmp/ruff.log &&
tail -n 20 /tmp/ruff.log
All checks passed!(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck# pyright >/tmp/pyright.log &&
tail -n 20 /tmp/pyright.log(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck# cat /tmp/pyright.log
/workspace/scubaduck/scubaduck/server.py
/workspace/scubaduck/scubaduck/server.py:434:13 - error: Type of "mn" is unkno
wn (reportUnknownVariableType)
/workspace/scubaduck/scubaduck/server.py:434:17 - error: Type of "mx" is unkno
wn (reportUnknownVariableType)
/workspace/scubaduck/scubaduck/server.py:434:22 - error: Type of "fetchone" is
unknown (reportUnknownMemberType)
/workspace/scubaduck/scubaduck/server.py:436:15 - error: Cannot access attribu
te "fetchone" for class "DuckDBPyRelation"
Attribute "fetchone" is unknown (reportAttributeAccessIssue)
/workspace/scubaduck/scubaduck/server.py:439:21 - error: Type of "strftime" is
unknown (reportUnknownMemberType)
/workspace/scubaduck/scubaduck/server.py:443:21 - error: Type of "strftime" is
unknown (reportUnknownMemberType)
/workspace/scubaduck/scubaduck/server.py:449:37 - error: Type of "start" is pa
rtially unknown
Type of "start" is "Unknown | str | None" (reportUnknownMemberType)
/workspace/scubaduck/scubaduck/server.py:449:37 - error: Argument type is part
ially unknown
Argument corresponds to parameter "start" in function "_granularity_seconds"
Argument type is "Unknown | str | None" (reportUnknownArgumentType)
/workspace/scubaduck/scubaduck/server.py:449:51 - error: Type of "end" is part
ially unknown
Type of "end" is "Unknown | str | None" (reportUnknownMemberType)
/workspace/scubaduck/scubaduck/server.py:449:51 - error: Argument type is part
ially unknown
Argument corresponds to parameter "end" in function "_granularity_seconds"
Argument type is "Unknown | str | None" (reportUnknownArgumentType)
/workspace/scubaduck/scubaduck/server.py:464:12 - error: Type of "start" is pa
rtially unknown
Type of "start" is "Unknown | str | None" (reportUnknownMemberType)
/workspace/scubaduck/scubaduck/server.py:465:31 - error: Type of "start" is pa
rtially unknown
Type of "start" is "Unknown | str" (reportUnknownMemberType)
/workspace/scubaduck/scubaduck/server.py:466:12 - error: Type of "end" is part
ially unknown
Type of "end" is "Unknown | str | None" (reportUnknownMemberType)
/workspace/scubaduck/scubaduck/server.py:467:29 - error: Type of "end" is part
ially unknown
Type of "end" is "Unknown | str" (reportUnknownMemberType)
/workspace/scubaduck/scubaduck/server.py:469:13 - error: Argument of type "int
" cannot be assigned to parameter "value" of type "str | list[tuple[Any, ...]]"
in function "__setitem__"
Type "int" is not assignable to type "str | list[tuple[Any, ...]]"
"int" is not assignable to "str"
"int" is not assignable to "list[tuple[Any, ...]]" (reportArgumentType)
15 errors, 0 warnings, 0 informations
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck#Pyright raised an issue due to untyped variables. Time to adjust or add annotations.
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck# nl -ba scubaduck/server.py |
sed -n '420,480p'
420 ),
421 400,
422 )
423 if allow_time and not (is_numeric or is_time):
424 return (
425 jsonify(
426 {
427 "error": f"Aggregate {agg} cannot be
applied to column {c}",
428 }
429 ),
430 400,
431 )
432 if params.start is None or params.end is None:
433 axis = params.x_axis or "timestamp"
434 mn, mx = con.execute(
435 f"SELECT min({axis}), max({axis}) FROM events"
436 ).fetchone()
437 if params.start is None and mn is not None:
438 params.start = (
439 mn.strftime("%Y-%m-%d %H:%M:%S") if not isinstance(m
n, str) else mn
440 )
441 if params.end is None and mx is not None:
442 params.end = (
443 mx.strftime("%Y-%m-%d %H:%M:%S") if not isinstance(m
x, str) else mx
444 )
445
446 bucket_size: int | None = None
447 if params.graph_type == "timeseries":
448 bucket_size = _granularity_seconds(
449 params.granularity, params.start, params.end
450 )
451
452 sql = build_query(params, column_types)
453 try:
454 rows = con.execute(sql).fetchall()
455 except Exception as exc:
456 tb = traceback.format_exc()
457 print(f"Query failed:\n{sql}\n{tb}")
458 return (
459 jsonify({"sql": sql, "error": str(exc), "traceback": tb}
),
460 400,
461 )
462
463 result = {"sql": sql, "rows": rows}
464 if params.start is not None:
465 result["start"] = params.start
466 if params.end is not None:
467 result["end"] = params.end
468 if bucket_size is not None:
469 result["bucket_size"] = bucket_size
470 return jsonify(result)
471
472 return app
473
474
475 app = create_app()
476
477
478 if __name__ == "__main__":
479 app.run(debug=True)
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck#To resolve Pyright issues, I'll adjust the types for mn and mx and cast them after .fetchone().
Pyright is complaining about bucket_size types, so I'll update result to dict[str, Any].
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck# apply_patch<<'PATCH'
> *** Begin Patch
> *** Update File: scubaduck/server.py
> @@
> - if params.start is None or params.end is None:
> - axis = params.x_axis or "timestamp"
> - mn, mx = con.execute(
> - f"SELECT min({axis}), max({axis}) FROM events"
> - ).fetchone()
> - if params.start is None and mn is not None:
> - params.start = (
> - mn.strftime("%Y-%m-%d %H:%M:%S") if not isinstance(mn, st
r) else mn
> - )
> - if params.end is None and mx is not None:
> - params.end = (
> - mx.strftime("%Y-%m-%d %H:%M:%S") if not isinstance(mx, st
r) else mx
> - )
> + if params.start is None or params.end is None:
> + axis = params.x_axis or "timestamp"
> + row: tuple[Any, Any] = (
> + con.execute(f"SELECT min({axis}), max({axis}) FROM events").f
etchone()
> + )
> + mn, mx = row
> + if params.start is None and mn is not None:
> + params.start = (
> + mn.strftime("%Y-%m-%d %H:%M:%S") if not isinstance(mn, st
r) else mn
> + )
> + if params.end is None and mx is not None:
> + params.end = (
> + mx.strftime("%Y-%m-%d %H:%M:%S") if not isinstance(mx, st
r) else mx
> + )
> *** End Patch
> PATCH
Success. Updated the following files:
M scubaduck/server.py
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck#I’ll update the patch to accommodate start and end as str | None types.
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck# apply_patch<<'PATCH'
> *** Begin Patch
> *** Update File: scubaduck/server.py
> @@
> - bucket_size: int | None = None
> - if params.graph_type == "timeseries":
> - bucket_size = _granularity_seconds(
> - params.granularity, params.start, params.end
> - )
> + bucket_size: int | None = None
> + if params.graph_type == "timeseries":
> + bucket_size = _granularity_seconds(
> + params.granularity,
> + params.start if isinstance(params.start, str) else None,
> + params.end if isinstance(params.end, str) else None,
> + )
> *** End Patch
> PATCH
Success. Updated the following files:
M scubaduck/server.py
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck#I’ll now update the result dictionary to include type annotations.
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck# apply_patch<<'PATCH'
> *** Begin Patch
> *** Update File: scubaduck/server.py
> @@
> - result = {"sql": sql, "rows": rows}
> + result: Dict[str, Any] = {"sql": sql, "rows": rows}
> *** End Patch
> PATCH
Success. Updated the following files:
M scubaduck/server.py
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck#I’ll check lines around 460 to 464 for the setitem typing error.
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck# nl -ba scubaduck/server.py |
sed -n '460,474p'
460 print(f"Query failed:\n{sql}\n{tb}")
461 return (
462 jsonify({"sql": sql, "error": str(exc), "traceback": tb}
),
463 400,
464 )
465
466 result: Dict[str, Any] = {"sql": sql, "rows": rows}
467 if params.start is not None:
468 result["start"] = params.start
469 if params.end is not None:
470 result["end"] = params.end
471 if bucket_size is not None:
472 result["bucket_size"] = bucket_size
473 return jsonify(result)
474
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck#Pyright's complaining about types. I'll cast the results to resolve the type mismatch.
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck# apply_patch<<'PATCH'
> *** Begin Patch
> *** Update File: scubaduck/server.py
> @@
> - if params.start is not None:
> - result["start"] = params.start
> - if params.end is not None:
> - result["end"] = params.end
> + if params.start is not None:
> + result["start"] = str(params.start)
> + if params.end is not None:
> + result["end"] = str(params.end)
> *** End Patch
> PATCH
Success. Updated the following files:
M scubaduck/server.py(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck# ruff format scubaduck/server
.py >/tmp/format.log && tail -n 20 /tmp/format.log
1 file reformatted
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck#I'll run the ruff check once more to ensure everything's in order.
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck# ruff check >/tmp/ruff.log &&
tail -n 20 /tmp/ruff.log
All checks passed!
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck#I'll give pyright another go. Let's run it again.
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck# pyright >/tmp/pyright.log &&
tail -n 20 /tmp/pyright.log(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck# cat /tmp/pyright.log
/workspace/scubaduck/scubaduck/server.py
/workspace/scubaduck/scubaduck/server.py:434:13 - error: Type of "row" is unkn
own (reportUnknownVariableType)
/workspace/scubaduck/scubaduck/server.py:434:36 - error: Type of "fetchone" is
unknown (reportUnknownMemberType)
/workspace/scubaduck/scubaduck/server.py:436:15 - error: Cannot access attribu
te "fetchone" for class "DuckDBPyRelation"
Attribute "fetchone" is unknown (reportAttributeAccessIssue)
/workspace/scubaduck/scubaduck/server.py:437:13 - error: Type of "mn" is parti
ally unknown
Type of "mn" is "Unknown | Any" (reportUnknownVariableType)
/workspace/scubaduck/scubaduck/server.py:437:17 - error: Type of "mx" is parti
ally unknown
Type of "mx" is "Unknown | Any" (reportUnknownVariableType)
/workspace/scubaduck/scubaduck/server.py:440:21 - error: Type of "strftime" is
partially unknown
Type of "strftime" is "Unknown | Any" (reportUnknownMemberType)
/workspace/scubaduck/scubaduck/server.py:444:21 - error: Type of "strftime" is
partially unknown
Type of "strftime" is "Unknown | Any" (reportUnknownMemberType)
/workspace/scubaduck/scubaduck/server.py:451:44 - error: Type of "start" is pa
rtially unknown
Type of "start" is "Unknown | str | None" (reportUnknownMemberType)
/workspace/scubaduck/scubaduck/server.py:452:42 - error: Type of "end" is part
ially unknown
Type of "end" is "Unknown | str | None" (reportUnknownMemberType)
/workspace/scubaduck/scubaduck/server.py:467:12 - error: Type of "start" is pa
rtially unknown
Type of "start" is "str | Unknown | None" (reportUnknownMemberType)
/workspace/scubaduck/scubaduck/server.py:468:35 - error: Type of "start" is pa
rtially unknown
Type of "start" is "str | Unknown" (reportUnknownMemberType)
/workspace/scubaduck/scubaduck/server.py:468:35 - error: Argument type is part
ially unknown
Argument corresponds to parameter "object" in function "__new__"
Argument type is "str | Unknown" (reportUnknownArgumentType)
/workspace/scubaduck/scubaduck/server.py:469:12 - error: Type of "end" is part
ially unknown
Type of "end" is "str | Unknown | None" (reportUnknownMemberType)
/workspace/scubaduck/scubaduck/server.py:470:33 - error: Type of "end" is part
ially unknown
Type of "end" is "str | Unknown" (reportUnknownMemberType)
/workspace/scubaduck/scubaduck/server.py:470:33 - error: Argument type is part
ially unknown
Argument corresponds to parameter "object" in function "__new__"
Argument type is "str | Unknown" (reportUnknownArgumentType)
15 errors, 0 warnings, 0 informations
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck#I'll update the code and cast the return type using typing.cast to silence Pyright errors.
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck# apply_patch<<'PATCH'
> *** Begin Patch
> *** Update File: scubaduck/server.py
> @@
> - row: tuple[Any, Any] = (
> - con.execute(f"SELECT min({axis}), max({axis}) FROM events").f
etchone()
> - )
> - mn, mx = row
> + row = cast(
> + tuple[datetime | None, datetime | None],
> + con.execute(f"SELECT min({axis}), max({axis}) FROM events").f
etchone(),
> + )
> + mn, mx = row
> *** End Patch
> PATCH
Failed to find expected lines [" row: tuple[Any, Any] = (", "
con.execute(f\"SELECT min({axis}), max({axis}) FROM events\").fetchone()
", " )", " mn, mx = row"] in scubaduck/server.py
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck#The patch failed due to quoting; I'll take a look at the snippet.
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck# sed -n '432,445p' scubaduck/
server.py
if params.start is None or params.end is None:
axis = params.x_axis or "timestamp"
row: tuple[Any, Any] = con.execute(
f"SELECT min({axis}), max({axis}) FROM events"
).fetchone()
mn, mx = row
if params.start is None and mn is not None:
params.start = (
mn.strftime("%Y-%m-%d %H:%M:%S") if not isinstance(mn, str)
else mn
)
if params.end is None and mx is not None:
params.end = (
mx.strftime("%Y-%m-%d %H:%M:%S") if not isinstance(mx, str)
else mx
)
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck#I'll fix the type by casting it properly instead of using a tuple.
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck# apply_patch<<'PATCH'
> *** Begin Patch
> *** Update File: scubaduck/server.py
> @@
> - row: tuple[Any, Any] = con.execute(
> - f"SELECT min({axis}), max({axis}) FROM events"
> - ).fetchone()
> + row = cast(
> + tuple[datetime | None, datetime | None],
> + con.execute(f"SELECT min({axis}), max({axis}) FROM events").f
etchone(),
> + )
> *** End Patch
> PATCH
Success. Updated the following files:
M scubaduck/server.py
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck#I'll patch lines with 'row:' and run ruff format one more time.
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck# ruff format scubaduck/server
.py >/tmp/format.log && tail -n 20 /tmp/format.log
1 file left unchanged(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck# ruff check >/tmp/ruff.log &&
tail -n 20 /tmp/ruff.log(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck# cat /tmp/ruff.log
scubaduck/server.py:434:19: F821 Undefined name `cast`
|
432 | if params.start is None or params.end is None:
433 | axis = params.x_axis or "timestamp"
434 | row = cast(
| ^^^^ F821
435 | tuple[datetime | None, datetime | None],
436 | con.execute(f"SELECT min({axis}), max({axis}) FROM events"
).fetchone(),
|
Found 1 error.
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck#I need to import cast from the typing module.
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck# grep -n "from typing" -n scu
baduck/server.py | head
4:from typing import Any, Dict, List, Tuple
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck#I'll update the code to include the necessary cast import.
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck# apply_patch<<'PATCH'
> *** Begin Patch
> *** Update File: scubaduck/server.py
> @@
> -from typing import Any, Dict, List, Tuple
> +from typing import Any, Dict, List, Tuple, cast
> *** End Patch
> PATCH
Success. Updated the following files:
M scubaduck/server.py(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck# ruff format scubaduck/server
.py >/tmp/format.log && tail -n 20 /tmp/format.log
1 file left unchanged
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck#I'll execute the ruff check again for validation.
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck# ruff check >/tmp/ruff.log &&
tail -n 20 /tmp/ruff.log
All checks passed!
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck#I'll run pyright one more time to ensure everything checks out.
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck# pyright >/tmp/pyright.log &&
tail -n 20 /tmp/pyright.log(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck# cat /tmp/pyright.log
/workspace/scubaduck/scubaduck/server.py
/workspace/scubaduck/scubaduck/server.py:436:17 - error: Type of "fetchone" is
unknown (reportUnknownMemberType)
/workspace/scubaduck/scubaduck/server.py:436:77 - error: Cannot access attribu
te "fetchone" for class "DuckDBPyRelation"
Attribute "fetchone" is unknown (reportAttributeAccessIssue)
2 errors, 0 warnings, 0 informations
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck#Instead of fetchone, I'll switch to fetchall()[0], which is better typed.
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck# grep -n "fetchone" -n scubad
uck/server.py
436: con.execute(f"SELECT min({axis}), max({axis}) FROM events").
fetchone(),(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck# apply_patch<<'PATCH'
> *** Begin Patch
> *** Update File: scubaduck/server.py
> @@
> - row = cast(
> - tuple[datetime | None, datetime | None],
> - con.execute(f"SELECT min({axis}), max({axis}) FROM events").f
etchone(),
> - )
> + row = cast(
> + tuple[datetime | None, datetime | None],
> + con.execute(f"SELECT min({axis}), max({axis}) FROM events").f
etchall()[0],
> + )
> *** End Patch
> PATCH
Success. Updated the following files:
M scubaduck/server.py
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck#I'll try running ruff format once more to ensure everything is properly formatted.
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck# ruff format scubaduck/server
.py >/tmp/format.log && tail -n 20 /tmp/format.log
1 file reformatted(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck# ruff check >/tmp/ruff.log &&
tail -n 20 /tmp/ruff.log
All checks passed!
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck#Time to give pyright another go. Let's try it one more time!
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck# pyright >/tmp/pyright.log &&
tail -n 20 /tmp/pyright.log0 errors, 0 warnings, 0 informations
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck#I'll open the first 60 lines of test_server.py to align with existing test styles.
from __future__ import annotations
import json
from pathlib import Path
import duckdb
from scubaduck import server
import pytest
def test_basic_query() -> None:
app = server.app
client = app.test_client()
payload = {
"start": "2024-01-01 00:00:00",
"end": "2024-01-02 00:00:00",
"order_by": "timestamp",
"order_dir": "ASC",
"limit": 10,
"columns": ["timestamp", "event", "value", "user"],
"filters": [],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert data
rows = data["rows"]
# We expect first three rows (until 2024-01-02 00:00:00)
assert len(rows) == 3
assert rows[0][1] == "login"
assert rows[1][1] == "logout"
def test_js_served() -> None:
app = server.app
client = app.test_client()
rv = client.get("/js/chip_input.js")
assert rv.status_code == 200
assert b"initChipInput" in rv.data
def test_filter_multi_token() -> None:
app = server.app
client = app.test_client()
payload = {
"start": "2024-01-01 00:00:00",
"end": "2024-01-02 03:00:00",
"order_by": "timestamp",
"limit": 10,
"columns": ["timestamp", "event", "value", "user"],
"filters": [{"column": "user", "op": "=", "value": ["alice", "charlie"]}
],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert data
rows = data["rows"]
# Should only return rows for alice and charlie
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck#I'll scroll to the bottom of the file to find the final test to patch.
.py
def test_timeseries_derived_column() -> None:
app = server.app
client = app.test_client()
payload = {
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "timeseries",
"granularity": "1 hour",
"limit": 7,
"columns": ["value"],
"derived_columns": {"derived_1": "value * 2"},
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
rows = data["rows"]
assert all(r[2] == r[1] * 2 for r in rows)
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck#I'll add a test to check the default start and end times after this. Let’s patch.
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck# apply_patch<<'PATCH'
> *** Begin Patch
> *** Update File: tests/test_server.py
> @@
> def test_timeseries_derived_column() -> None:
> @@
> assert rv.status_code == 200
> rows = data["rows"]
> assert all(r[2] == r[1] * 2 for r in rows)
> +
> +
> +def test_default_start_end_returned() -> None:
> + app = server.app
> + client = app.test_client()
> + payload = {
> + "order_by": "timestamp",
> + "limit": 5,
> + "columns": ["timestamp"],
> + }
> + rv = client.post(
> + "/api/query", data=json.dumps(payload), content_type="application/jso
n"
> + )
> + data = rv.get_json()
> + assert rv.status_code == 200
> + assert data["start"] == "2024-01-01 00:00:00"
> + assert data["end"] == "2024-01-02 03:00:00"
> *** End Patch
> PATCH
Success. Updated the following files:
M tests/test_server.py
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck#I'll search for the timeseries_single_bucket test and add a new one for fill.
ucket" -n tests/test_web.py
213:def test_timeseries_single_bucket(page: Any, server_url: str) -> None:
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck#I'm looking at the code starting from around line 210.
_web.py
assert not page.is_checked("#column_groups input[value='timestamp']")
def test_timeseries_single_bucket(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#graph_type", state="attached")
page.fill("#start", "2024-01-01 00:00:00")
page.fill("#end", "2024-01-01 00:00:00")
select_value(page, "#graph_type", "timeseries")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
path = page.get_attribute("#chart path", "d")
assert path is not None and "NaN" not in path
def test_help_and_alignment(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
titles = page.evaluate(
"Array.from(document.querySelectorAll('#settings .help')).map(e => e.tit
le)"
)
assert any("start/end of the time range" in t for t in titles)
text_align = page.evaluate(
"getComputedStyle(document.querySelector('#settings label')).textAlign"
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck#I'll add a new test after the test_timeseries_single_bucket, following the plan.
> *** Update File: tests/test_web.py
> @@
> def test_timeseries_single_bucket(page: Any, server_url: str) -> None:
> @@
> assert path is not None and "NaN" not in path
> +
> +
> +def test_timeseries_fill_options(page: Any, server_url: str) -> None:
> + page.goto(server_url)
> + page.wait_for_selector("#graph_type", state="attached")
> + page.fill("#start", "2024-01-01 00:00:00")
> + page.fill("#end", "2024-01-02 03:00:00")
> + select_value(page, "#graph_type", "timeseries")
> + select_value(page, "#granularity", "1 hour")
> +
> + select_value(page, "#fill", "0")
> + page.evaluate("window.lastResults = undefined")
> + page.click("text=Dive")
> + page.wait_for_function("window.lastResults !== undefined")
> + path_zero = page.get_attribute("#chart path", "d")
> + assert path_zero is not None and path_zero.count("L") > 20
> +
> + select_value(page, "#fill", "connect")
> + page.evaluate("window.lastResults = undefined")
> + page.click("text=Dive")
> + page.wait_for_function("window.lastResults !== undefined")
> + path_conn = page.get_attribute("#chart path", "d")
> + assert path_conn is not None and path_conn.count("M") == 1
> +
> + select_value(page, "#fill", "blank")
> + page.evaluate("window.lastResults = undefined")
> + page.click("text=Dive")
> + page.wait_for_function("window.lastResults !== undefined")
> + path_blank = page.get_attribute("#chart path", "d")
> + assert path_blank is not None and path_blank.count("M") > 1
> *** End Patch
> PATCH
Success. Updated the following files:
M tests/test_web.py
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck#I'll fix the patch to follow PEP8 with ruff but skip HTML/JS files.
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck# ruff format tests/test_serve
r.py tests/test_web.py >/tmp/format.log && tail -n 20 /tmp/format.log
2 files left unchanged
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck#I'll take another quick look to make sure everything checks out.
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck# ruff check >/tmp/ruff.log &&
tail -n 20 /tmp/ruff.log
All checks passed!(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck# pyright >/tmp/pyright.log &&
tail -n 20 /tmp/pyright.log0 errors, 0 warnings, 0 informations
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck#The tests are up next—time to run pytest and check for any issues!
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck# pytest -q >/tmp/pytest.log &
& tail -n 20 /tmp/pytest.logbringing up nodes...
bringing up nodes...
..................................................................... [100%]
69 passed in 11.73s(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck# cat /tmp/pytest.log
bringing up nodes...
bringing up nodes...
..................................................................... [100%]
69 passed in 11.73s
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck#All tests passed successfully—time to commit the changes now.
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck# git status --short
M scubaduck/server.py
M scubaduck/static/js/timeseries_chart.js
M tests/test_server.py
M tests/test_web.py(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck# git add scubaduck/server.py
scubaduck/static/js/timeseries_chart.js tests/test_server.py tests/test_web.py(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck# git commit -m "feat(timeseri
es): handle bucket gaps and auto-range" && git status --short
[work 59a3820] feat(timeseries): handle bucket gaps and auto-range
4 files changed, 153 insertions(+), 18 deletions(-)
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck#I mistakenly didn't truncate the commit message. Let's check the status now.
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck# git status --short
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck#I'll gather the code lines from server.py referencing the new start and end features.
428 }
429 ),
430 400,
431 )
432 if params.start is None or params.end is None:
433 axis = params.x_axis or "timestamp"
434 row = cast(
435 tuple[datetime | None, datetime | None],
436 con.execute(f"SELECT min({axis}), max({axis}) FROM event
s").fetchall()[
437 0
438 ],
439 )
440 mn, mx = row
441 if params.start is None and mn is not None:
442 params.start = (
443 mn.strftime("%Y-%m-%d %H:%M:%S") if not isinstance(m
n, str) else mn
444 )
445 if params.end is None and mx is not None:
446 params.end = (
447 mx.strftime("%Y-%m-%d %H:%M:%S") if not isinstance(m
x, str) else mx
448 )
449
450 bucket_size: int | None = None
451 if params.graph_type == "timeseries":
452 bucket_size = _granularity_seconds(
453 params.granularity,
454 params.start if isinstance(params.start, str) else None,
455 params.end if isinstance(params.end, str) else None,
456 )
457
458 sql = build_query(params, column_types)
459 try:
460 rows = con.execute(sql).fetchall()
461 except Exception as exc:
462 tb = traceback.format_exc()
463 print(f"Query failed:\n{sql}\n{tb}")
464 return (
465 jsonify({"sql": sql, "error": str(exc), "traceback": tb}
),
466 400,
467 )
468
469 result: Dict[str, Any] = {"sql": sql, "rows": rows}
470 if params.start is not None:
471 result["start"] = str(params.start)
472 if params.end is not None:(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck# nl -ba scubaduck/server.py |
sed -n '472,486p'
472 if params.end is not None:
473 result["end"] = str(params.end)
474 if bucket_size is not None:
475 result["bucket_size"] = bucket_size
476 return jsonify(result)
477
478 return app
479
480
481 app = create_app()
482
483
484 if __name__ == "__main__":
485 app.run(debug=True)
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck#I'll gather the JavaScript lines that reference 'fill' and related terms.
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck# nl -ba scubaduck/static/js/t
imeseries_chart.js | sed -n '1,140p'
1 function showTimeSeries(data) {
2 const view = document.getElementById('view');
3 if (data.rows.length === 0) {
4 view.innerHTML = '<p id="empty-message">Empty data provided to table
</p>';
5 return;
6 }
7 const width = 600;
8 const height = 400;
9 view.innerHTML =
10 '<div id="legend"></div><svg id="chart" width="' +
11 width +
12 '" height="' +
13 height +
14 '"></svg>';
15 const svg = document.getElementById('chart');
16 const legend = document.getElementById('legend');
17 const groups = groupBy.chips || [];
18 const hasHits = document.getElementById('show_hits').checked ? 1 : 0;
19 const fill = document.getElementById('fill').value;
20 const bucketMs = (data.bucket_size || 3600) * 1000;
21 const start = data.start ? new Date(data.start).getTime() : null;
22 const end = data.end ? new Date(data.end).getTime() : null;
23 const series = {};
24 data.rows.forEach(r => {
25 const ts = new Date(r[0]).getTime();
26 const key = groups.map((_, i) => r[1 + i]).join(':') || 'all';
27 const val = Number(r[1 + groups.length + hasHits]);
28 if (!series[key]) series[key] = {};
29 series[key][ts] = val;
30 });
31
32 const buckets = [];
33 let minX = start !== null ? start : Infinity;
34 let maxX = end !== null ? end : -Infinity;
35 if (start !== null && end !== null) {
36 for (let t = start; t <= end; t += bucketMs) {
37 buckets.push(t);
38 }
39 } else {
40 Object.keys(series).forEach(k => {
41 const s = series[k];
42 Object.keys(s).forEach(t => {
43 const n = Number(t);
44 if (n < minX) minX = n;
45 if (n > maxX) maxX = n;
46 });
47 });
48 for (let t = minX; t <= maxX; t += bucketMs) {
49 buckets.push(t);
50 }
51 }
52
53 let minY = Infinity,
54 maxY = -Infinity;
55 Object.keys(series).forEach(key => {
56 const vals = series[key];
57 buckets.forEach(b => {
58 const v = vals[b];
59 const val = v === undefined && fill === '0' ? 0 : v;
60 if (val === undefined) return;
61 if (val < minY) minY = val;
62 if (val > maxY) maxY = val;
63 });
64 });
65 if (fill === '0') {
66 if (minY > 0) minY = 0;
67 if (maxY < 0) maxY = 0;
68 }
69
70 const colors = [
71 '#1f77b4',
72 '#ff7f0e',
73 '#2ca02c',
74 '#d62728',
75 '#9467bd',
76 '#8c564b',
77 '#e377c2'
78 ];
79 let colorIndex = 0;
80 const xRange = maxX - minX || 1;
81 const yRange = maxY - minY || 1;
82 const xScale = x => ((x - minX) / xRange) * (width - 60) + 50;
83 const yScale = y => height - 30 - ((y - minY) / yRange) * (height - 60
);
84
85 Object.keys(series).forEach(key => {
86 const vals = series[key];
87 const color = colors[colorIndex++ % colors.length];
88 let path = '';
89 let drawing = false;
90 buckets.forEach(b => {
91 const v = vals[b];
92 if (v === undefined) {
93 if (fill === '0') {
94 const x = xScale(b);
95 const y = yScale(0);
96 path += (drawing ? 'L' : 'M') + x + ' ' + y + ' ';
97 drawing = true;
98 } else if (fill === 'blank') {
99 drawing = false;
100 }
101 // connect: do nothing
102 } else {
103 const x = xScale(b);
104 const y = yScale(v);
105 path += (drawing ? 'L' : 'M') + x + ' ' + y + ' ';
106 drawing = true;
107 }
108 });
109 const el = document.createElementNS('http://www.w3.org/2000/svg', 'p
ath');
110 el.setAttribute('d', path.trim());
111 el.setAttribute('fill', 'none');
112 el.setAttribute('stroke', color);
113 svg.appendChild(el);
114 const item = document.createElement('div');
115 item.textContent = key;
116 item.style.color = color;
117 legend.appendChild(item);
118 });
119 }
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck#I'm now displaying the test lines for the server test.
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck# nl -ba tests/test_server.py
| sed -n '214,244p'
214 assert len(rows) == 1
215
216
217 def test_group_by_table() -> None:
218 app = server.app
219 client = app.test_client()
220 payload = {
221 "start": "2024-01-01 00:00:00",
222 "end": "2024-01-03 00:00:00",
223 "graph_type": "table",
224 "order_by": "user",
225 "limit": 10,
226 "columns": ["value"],
227 "group_by": ["user"],
228 "aggregate": "Sum",
229 "show_hits": True,
230 }
231 rv = client.post(
232 "/api/query", data=json.dumps(payload), content_type="applicatio
n/json"
233 )
234 rows = rv.get_json()["rows"]
235 assert rows[0][0] == "alice"
236 assert rows[0][1] == 2
237 assert rows[0][2] == 40
238
239
240 def test_relative_time_query(monkeypatch: pytest.MonkeyPatch) -> None:
241 app = server.app
242 client = app.test_client()
243
244 from datetime import datetime
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck#I’m scrolling down to highlight the lines I’ve added.
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck# tail -n 20 tests/test_server
.py
assert rv.status_code == 200
rows = data["rows"]
assert all(r[2] == r[1] * 2 for r in rows)
def test_default_start_end_returned() -> None:
app = server.app
client = app.test_client()
payload = {
"order_by": "timestamp",
"limit": 5,
"columns": ["timestamp"],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert data["start"] == "2024-01-01 00:00:00"
assert data["end"] == "2024-01-02 03:00:00"
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck#I’ll display the lines from test_web including the new test I just added.
ed -n '210,250p'
210 assert not page.is_checked("#column_groups input[value='timestamp']"
)
211
212
213 def test_timeseries_single_bucket(page: Any, server_url: str) -> None:
214 page.goto(server_url)
215 page.wait_for_selector("#graph_type", state="attached")
216 page.fill("#start", "2024-01-01 00:00:00")
217 page.fill("#end", "2024-01-01 00:00:00")
218 select_value(page, "#graph_type", "timeseries")
219 page.evaluate("window.lastResults = undefined")
220 page.click("text=Dive")
221 page.wait_for_function("window.lastResults !== undefined")
222 path = page.get_attribute("#chart path", "d")
223 assert path is not None and "NaN" not in path
224
225
226 def test_timeseries_fill_options(page: Any, server_url: str) -> None:
227 page.goto(server_url)
228 page.wait_for_selector("#graph_type", state="attached")
229 page.fill("#start", "2024-01-01 00:00:00")
230 page.fill("#end", "2024-01-02 03:00:00")
231 select_value(page, "#graph_type", "timeseries")
232 select_value(page, "#granularity", "1 hour")
233
234 select_value(page, "#fill", "0")
235 page.evaluate("window.lastResults = undefined")
236 page.click("text=Dive")
237 page.wait_for_function("window.lastResults !== undefined")
238 path_zero = page.get_attribute("#chart path", "d")
239 assert path_zero is not None and path_zero.count("L") > 20
240
241 select_value(page, "#fill", "connect")
242 page.evaluate("window.lastResults = undefined")
243 page.click("text=Dive")
244 page.wait_for_function("window.lastResults !== undefined")
245 path_conn = page.get_attribute("#chart path", "d")
246 assert path_conn is not None and path_conn.count("M") == 1
247
248 select_value(page, "#fill", "blank")
249 page.evaluate("window.lastResults = undefined")
250 page.click("text=Dive")(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck# nl -ba tests/test_web.py | s
ed -n '250,275p'
250 page.click("text=Dive")
251 page.wait_for_function("window.lastResults !== undefined")
252 path_blank = page.get_attribute("#chart path", "d")
253 assert path_blank is not None and path_blank.count("M") > 1
254
255
256 def test_help_and_alignment(page: Any, server_url: str) -> None:
257 page.goto(server_url)
258 page.wait_for_selector("#order_by option", state="attached")
259 titles = page.evaluate(
260 "Array.from(document.querySelectorAll('#settings .help')).map(e
=> e.title)"
261 )
262 assert any("start/end of the time range" in t for t in titles)
263
264 text_align = page.evaluate(
265 "getComputedStyle(document.querySelector('#settings label')).tex
tAlign"
266 )
267 assert text_align == "right"
268
269
270 def test_table_sorting(page: Any, server_url: str) -> None:
271 run_query(
272 page,
273 server_url,
274 start="2024-01-01 00:00:00",
275 end="2024-01-03 00:00:00",
(scubaduck) root@60bb5aa46aa5:/workspace/scubaduck#I'm drafting the final PR message now.