Task ID: task_e_682e2266425c8323a897024ebb341193
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (2/33)
pyee ------------------------------ 15.36 KiB/15.36 KiB
itsdangerous ------------------------------ 15.85 KiB/15.85 KiB
pluggy ------------------------------ 20.06 KiB/20.06 KiB
execnet ------------------------------ 30.88 KiB/39.66 KiB
pytest-xdist ------------------------------ 30.90 KiB/45.03 KiB
requests ------------------------------ 63.41 KiB/63.41 KiB
packaging ------------------------------ 46.88 KiB/64.91 KiB
click ------------------------------ 99.76 KiB/99.76 KiB
flask ------------------------------ 62.88 KiB/100.88 KiB
urllib3 ------------------------------ 77.71 KiB/125.66 KiB
jinja2 ------------------------------ 109.79 KiB/131.74 KiB
charset-normalizer ------------------------------ 125.00 KiB/145.08 KiB
werkzeug ------------------------------ 75.73 KiB/219.24 KiB
python-dateutil ------------------------------ 46.86 KiB/224.50 KiB
pytest ------------------------------ 48.00 KiB/335.58 KiB
greenlet ------------------------------ 32.00 KiB/589.71 KiB
pyright ------------------------------ 16.00 KiB/5.31 MiB
ruff ------------------------------ 349.47 KiB/11.02 MiB
duckdb ------------------------------ 213.61 KiB/19.27 MiB
playwright ------------------------------ 30.06 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (2/33)
pyee ------------------------------ 15.36 KiB/15.36 KiB
itsdangerous ------------------------------ 15.85 KiB/15.85 KiB
pluggy ------------------------------ 20.06 KiB/20.06 KiB
execnet ------------------------------ 30.88 KiB/39.66 KiB
pytest-xdist ------------------------------ 30.90 KiB/45.03 KiB
requests ------------------------------ 63.41 KiB/63.41 KiB
packaging ------------------------------ 46.88 KiB/64.91 KiB
flask ------------------------------ 62.88 KiB/100.88 KiB
urllib3 ------------------------------ 77.71 KiB/125.66 KiB
jinja2 ------------------------------ 125.79 KiB/131.74 KiB
charset-normalizer ------------------------------ 125.00 KiB/145.08 KiB
werkzeug ------------------------------ 75.73 KiB/219.24 KiB
python-dateutil ------------------------------ 46.86 KiB/224.50 KiB
pytest ------------------------------ 48.00 KiB/335.58 KiB
greenlet ------------------------------ 32.00 KiB/589.71 KiB
pyright ------------------------------ 16.00 KiB/5.31 MiB
ruff ------------------------------ 365.47 KiB/11.02 MiB
duckdb ------------------------------ 229.61 KiB/19.27 MiB
playwright ------------------------------ 30.06 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (2/33)
pyee ------------------------------ 15.36 KiB/15.36 KiB
pluggy ------------------------------ 20.06 KiB/20.06 KiB
execnet ------------------------------ 30.88 KiB/39.66 KiB
pytest-xdist ------------------------------ 30.90 KiB/45.03 KiB
requests ------------------------------ 63.41 KiB/63.41 KiB
packaging ------------------------------ 46.88 KiB/64.91 KiB
flask ------------------------------ 62.88 KiB/100.88 KiB
urllib3 ------------------------------ 77.71 KiB/125.66 KiB
jinja2 ------------------------------ 125.79 KiB/131.74 KiB
charset-normalizer ------------------------------ 125.00 KiB/145.08 KiB
werkzeug ------------------------------ 75.73 KiB/219.24 KiB
python-dateutil ------------------------------ 46.86 KiB/224.50 KiB
pytest ------------------------------ 48.00 KiB/335.58 KiB
greenlet ------------------------------ 32.00 KiB/589.71 KiB
pyright ------------------------------ 16.00 KiB/5.31 MiB
ruff ------------------------------ 365.47 KiB/11.02 MiB
duckdb ------------------------------ 229.61 KiB/19.27 MiB
playwright ------------------------------ 30.06 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (2/33)
pyee ------------------------------ 15.36 KiB/15.36 KiB
pluggy ------------------------------ 20.06 KiB/20.06 KiB
execnet ------------------------------ 30.88 KiB/39.66 KiB
pytest-xdist ------------------------------ 30.90 KiB/45.03 KiB
packaging ------------------------------ 46.88 KiB/64.91 KiB
flask ------------------------------ 62.88 KiB/100.88 KiB
urllib3 ------------------------------ 77.71 KiB/125.66 KiB
jinja2 ------------------------------ 125.79 KiB/131.74 KiB
charset-normalizer ------------------------------ 125.00 KiB/145.08 KiB
werkzeug ------------------------------ 75.73 KiB/219.24 KiB
python-dateutil ------------------------------ 46.86 KiB/224.50 KiB
pytest ------------------------------ 48.00 KiB/335.58 KiB
greenlet ------------------------------ 32.00 KiB/589.71 KiB
pyright ------------------------------ 16.00 KiB/5.31 MiB
ruff ------------------------------ 365.47 KiB/11.02 MiB
duckdb ------------------------------ 229.61 KiB/19.27 MiB
playwright ------------------------------ 30.06 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (2/33)
pyee ------------------------------ 15.36 KiB/15.36 KiB
execnet ------------------------------ 30.88 KiB/39.66 KiB
pytest-xdist ------------------------------ 30.90 KiB/45.03 KiB
packaging ------------------------------ 46.88 KiB/64.91 KiB
flask ------------------------------ 62.88 KiB/100.88 KiB
urllib3 ------------------------------ 93.71 KiB/125.66 KiB
jinja2 ------------------------------ 125.79 KiB/131.74 KiB
charset-normalizer ------------------------------ 141.00 KiB/145.08 KiB
werkzeug ------------------------------ 75.73 KiB/219.24 KiB
python-dateutil ------------------------------ 62.86 KiB/224.50 KiB
pytest ------------------------------ 48.00 KiB/335.58 KiB
greenlet ------------------------------ 32.00 KiB/589.71 KiB
pyright ------------------------------ 16.00 KiB/5.31 MiB
ruff ------------------------------ 397.47 KiB/11.02 MiB
duckdb ------------------------------ 261.61 KiB/19.27 MiB
playwright ------------------------------ 30.06 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (2/33)
execnet ------------------------------ 30.88 KiB/39.66 KiB
pytest-xdist ------------------------------ 45.03 KiB/45.03 KiB
packaging ------------------------------ 62.88 KiB/64.91 KiB
flask ------------------------------ 62.88 KiB/100.88 KiB
urllib3 ------------------------------ 93.71 KiB/125.66 KiB
jinja2 ------------------------------ 125.79 KiB/131.74 KiB
charset-normalizer ------------------------------ 141.00 KiB/145.08 KiB
werkzeug ------------------------------ 91.73 KiB/219.24 KiB
python-dateutil ------------------------------ 74.98 KiB/224.50 KiB
pytest ------------------------------ 62.78 KiB/335.58 KiB
greenlet ------------------------------ 48.00 KiB/589.71 KiB
pyright ------------------------------ 32.00 KiB/5.31 MiB
ruff ------------------------------ 413.47 KiB/11.02 MiB
duckdb ------------------------------ 277.61 KiB/19.27 MiB
playwright ------------------------------ 30.06 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (2/33)
execnet ------------------------------ 39.66 KiB/39.66 KiB
pytest-xdist ------------------------------ 45.03 KiB/45.03 KiB
packaging ------------------------------ 64.91 KiB/64.91 KiB
flask ------------------------------ 78.88 KiB/100.88 KiB
urllib3 ------------------------------ 93.71 KiB/125.66 KiB
jinja2 ------------------------------ 131.74 KiB/131.74 KiB
charset-normalizer ------------------------------ 145.08 KiB/145.08 KiB
werkzeug ------------------------------ 107.73 KiB/219.24 KiB
python-dateutil ------------------------------ 218.98 KiB/224.50 KiB
pytest ------------------------------ 110.78 KiB/335.58 KiB
greenlet ------------------------------ 93.89 KiB/589.71 KiB
pyright ------------------------------ 239.73 KiB/5.31 MiB
ruff ------------------------------ 605.47 KiB/11.02 MiB
duckdb ------------------------------ 485.61 KiB/19.27 MiB
playwright ------------------------------ 46.06 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (2/33)
execnet ------------------------------ 39.66 KiB/39.66 KiB
pytest-xdist ------------------------------ 45.03 KiB/45.03 KiB
flask ------------------------------ 78.88 KiB/100.88 KiB
urllib3 ------------------------------ 93.71 KiB/125.66 KiB
jinja2 ------------------------------ 131.74 KiB/131.74 KiB
charset-normalizer ------------------------------ 145.08 KiB/145.08 KiB
werkzeug ------------------------------ 107.73 KiB/219.24 KiB
python-dateutil ------------------------------ 218.98 KiB/224.50 KiB
pytest ------------------------------ 110.78 KiB/335.58 KiB
greenlet ------------------------------ 93.89 KiB/589.71 KiB
pyright ------------------------------ 239.73 KiB/5.31 MiB
ruff ------------------------------ 621.47 KiB/11.02 MiB
duckdb ------------------------------ 485.61 KiB/19.27 MiB
playwright ------------------------------ 46.06 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (2/33)
execnet ------------------------------ 39.66 KiB/39.66 KiB
pytest-xdist ------------------------------ 45.03 KiB/45.03 KiB
flask ------------------------------ 94.88 KiB/100.88 KiB
urllib3 ------------------------------ 109.71 KiB/125.66 KiB
charset-normalizer ------------------------------ 145.08 KiB/145.08 KiB
werkzeug ------------------------------ 107.73 KiB/219.24 KiB
python-dateutil ------------------------------ 218.98 KiB/224.50 KiB
pytest ------------------------------ 110.78 KiB/335.58 KiB
greenlet ------------------------------ 141.89 KiB/589.71 KiB
pyright ------------------------------ 271.73 KiB/5.31 MiB
ruff ------------------------------ 653.47 KiB/11.02 MiB
duckdb ------------------------------ 533.61 KiB/19.27 MiB
playwright ------------------------------ 46.06 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (2/33)
execnet ------------------------------ 39.66 KiB/39.66 KiB
flask ------------------------------ 94.88 KiB/100.88 KiB
urllib3 ------------------------------ 109.71 KiB/125.66 KiB
charset-normalizer ------------------------------ 145.08 KiB/145.08 KiB
werkzeug ------------------------------ 107.73 KiB/219.24 KiB
python-dateutil ------------------------------ 224.50 KiB/224.50 KiB
pytest ------------------------------ 110.78 KiB/335.58 KiB
greenlet ------------------------------ 157.89 KiB/589.71 KiB
pyright ------------------------------ 303.73 KiB/5.31 MiB
ruff ------------------------------ 685.47 KiB/11.02 MiB
duckdb ------------------------------ 549.61 KiB/19.27 MiB
playwright ------------------------------ 62.06 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (2/33)
flask ------------------------------ 100.88 KiB/100.88 KiB
urllib3 ------------------------------ 109.71 KiB/125.66 KiB
charset-normalizer ------------------------------ 145.08 KiB/145.08 KiB
werkzeug ------------------------------ 107.73 KiB/219.24 KiB
python-dateutil ------------------------------ 224.50 KiB/224.50 KiB
pytest ------------------------------ 126.78 KiB/335.58 KiB
greenlet ------------------------------ 173.89 KiB/589.71 KiB
pyright ------------------------------ 319.73 KiB/5.31 MiB
ruff ------------------------------ 701.47 KiB/11.02 MiB
duckdb ------------------------------ 565.61 KiB/19.27 MiB
playwright ------------------------------ 62.06 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (2/33)
flask ------------------------------ 100.88 KiB/100.88 KiB
urllib3 ------------------------------ 109.71 KiB/125.66 KiB
werkzeug ------------------------------ 123.73 KiB/219.24 KiB
python-dateutil ------------------------------ 224.50 KiB/224.50 KiB
pytest ------------------------------ 126.78 KiB/335.58 KiB
greenlet ------------------------------ 189.89 KiB/589.71 KiB
pyright ------------------------------ 335.73 KiB/5.31 MiB
ruff ------------------------------ 717.47 KiB/11.02 MiB
duckdb ------------------------------ 581.61 KiB/19.27 MiB
playwright ------------------------------ 62.06 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (2/33)
flask ------------------------------ 100.88 KiB/100.88 KiB
urllib3 ------------------------------ 125.66 KiB/125.66 KiB
werkzeug ------------------------------ 123.73 KiB/219.24 KiB
pytest ------------------------------ 142.78 KiB/335.58 KiB
greenlet ------------------------------ 269.89 KiB/589.71 KiB
pyright ------------------------------ 399.73 KiB/5.31 MiB
ruff ------------------------------ 797.47 KiB/11.02 MiB
duckdb ------------------------------ 661.61 KiB/19.27 MiB
playwright ------------------------------ 62.06 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (2/33)
urllib3 ------------------------------ 125.66 KiB/125.66 KiB
werkzeug ------------------------------ 139.73 KiB/219.24 KiB
pytest ------------------------------ 174.78 KiB/335.58 KiB
greenlet ------------------------------ 317.89 KiB/589.71 KiB
pyright ------------------------------ 463.73 KiB/5.31 MiB
ruff ------------------------------ 845.47 KiB/11.02 MiB
duckdb ------------------------------ 725.61 KiB/19.27 MiB
playwright ------------------------------ 62.06 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (2/33)
urllib3 ------------------------------ 125.66 KiB/125.66 KiB
werkzeug ------------------------------ 139.73 KiB/219.24 KiB
pytest ------------------------------ 190.78 KiB/335.58 KiB
greenlet ------------------------------ 413.89 KiB/589.71 KiB
pyright ------------------------------ 575.73 KiB/5.31 MiB
ruff ------------------------------ 941.47 KiB/11.02 MiB
duckdb ------------------------------ 805.61 KiB/19.27 MiB
playwright ------------------------------ 75.77 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (2/33)
werkzeug ------------------------------ 139.73 KiB/219.24 KiB
pytest ------------------------------ 190.78 KiB/335.58 KiB
greenlet ------------------------------ 445.89 KiB/589.71 KiB
pyright ------------------------------ 607.73 KiB/5.31 MiB
ruff ------------------------------ 973.47 KiB/11.02 MiB
duckdb ------------------------------ 837.61 KiB/19.27 MiB
playwright ------------------------------ 75.77 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (2/33)
werkzeug ------------------------------ 187.73 KiB/219.24 KiB
pytest ------------------------------ 238.78 KiB/335.58 KiB
greenlet ------------------------------ 477.89 KiB/589.71 KiB
pyright ------------------------------ 1.09 MiB/5.31 MiB
ruff ------------------------------ 1.45 MiB/11.02 MiB
duckdb ------------------------------ 1.29 MiB/19.27 MiB
playwright ------------------------------ 107.77 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠸ Preparing packages... (25/33)
pytest ------------------------------ 302.78 KiB/335.58 KiB
greenlet ------------------------------ 493.89 KiB/589.71 KiB
pyright ------------------------------ 1.33 MiB/5.31 MiB
ruff ------------------------------ 1.80 MiB/11.02 MiB
duckdb ------------------------------ 1.64 MiB/19.27 MiB
playwright ------------------------------ 283.77 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠸ Preparing packages... (25/33)
pytest ------------------------------ 318.78 KiB/335.58 KiB
greenlet ------------------------------ 493.89 KiB/589.71 KiB
pyright ------------------------------ 1.55 MiB/5.31 MiB
ruff ------------------------------ 2.04 MiB/11.02 MiB
duckdb ------------------------------ 1.89 MiB/19.27 MiB
playwright ------------------------------ 539.77 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠸ Preparing packages... (25/33)
greenlet ------------------------------ 541.89 KiB/589.71 KiB
pyright ------------------------------ 1.78 MiB/5.31 MiB
ruff ------------------------------ 2.36 MiB/11.02 MiB
duckdb ------------------------------ 2.20 MiB/19.27 MiB
playwright ------------------------------ 859.77 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠸ Preparing packages... (25/33)
greenlet ------------------------------ 557.89 KiB/589.71 KiB
pyright ------------------------------ 1.80 MiB/5.31 MiB
ruff ------------------------------ 2.75 MiB/11.02 MiB
duckdb ------------------------------ 2.59 MiB/19.27 MiB
playwright ------------------------------ 1.22 MiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠸ Preparing packages... (25/33)
pyright ------------------------------ 1.81 MiB/5.31 MiB
ruff ------------------------------ 3.39 MiB/11.02 MiB
duckdb ------------------------------ 3.23 MiB/19.27 MiB
playwright ------------------------------ 1.84 MiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠸ Preparing packages... (25/33)
pyright ------------------------------ 1.81 MiB/5.31 MiB
ruff ------------------------------ 3.45 MiB/11.02 MiB
duckdb ------------------------------ 3.30 MiB/19.27 MiB
playwright ------------------------------ 1.92 MiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠸ Preparing packages... (25/33)
pyright ------------------------------ 1.83 MiB/5.31 MiB
ruff ------------------------------ 3.83 MiB/11.02 MiB
duckdb ------------------------------ 3.70 MiB/19.27 MiB
playwright ------------------------------ 2.31 MiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠼ Preparing packages... (28/33)
pyright ------------------------------ 1.84 MiB/5.31 MiB
ruff ------------------------------ 4.28 MiB/11.02 MiB
duckdb ------------------------------ 4.14 MiB/19.27 MiB
playwright ------------------------------ 2.75 MiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠼ Preparing packages... (28/33)
pyright ------------------------------ 1.87 MiB/5.31 MiB
ruff ------------------------------ 4.67 MiB/11.02 MiB
duckdb ------------------------------ 4.51 MiB/19.27 MiB
playwright ------------------------------ 3.15 MiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠼ Preparing packages... (28/33)
pyright ------------------------------ 1.91 MiB/5.31 MiB
ruff ------------------------------ 5.15 MiB/11.02 MiB
duckdb ------------------------------ 5.00 MiB/19.27 MiB
playwright ------------------------------ 3.62 MiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠼ Preparing packages... (28/33)
pyright ------------------------------ 1.94 MiB/5.31 MiB
ruff ------------------------------ 5.93 MiB/11.02 MiB
duckdb ------------------------------ 5.78 MiB/19.27 MiB
playwright ------------------------------ 4.40 MiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠴ Preparing packages... (28/33)
pyright ------------------------------ 1.97 MiB/5.31 MiB
ruff ------------------------------ 6.70 MiB/11.02 MiB
duckdb ------------------------------ 6.56 MiB/19.27 MiB
playwright ------------------------------ 5.20 MiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠴ Preparing packages... (28/33)
pyright ------------------------------ 2.00 MiB/5.31 MiB
ruff ------------------------------ 7.39 MiB/11.02 MiB
duckdb ------------------------------ 7.26 MiB/19.27 MiB
playwright ------------------------------ 5.90 MiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠴ Preparing packages... (28/33)
pyright ------------------------------ 2.03 MiB/5.31 MiB
ruff ------------------------------ 8.01 MiB/11.02 MiB
duckdb ------------------------------ 7.89 MiB/19.27 MiB
playwright ------------------------------ 6.50 MiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠴ Preparing packages... (28/33)
pyright ------------------------------ 2.08 MiB/5.31 MiB
ruff ------------------------------ 8.66 MiB/11.02 MiB
duckdb ------------------------------ 8.53 MiB/19.27 MiB
playwright ------------------------------ 7.17 MiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠦ Preparing packages... (28/33)
pyright ------------------------------ 2.09 MiB/5.31 MiB
ruff ------------------------------ 9.33 MiB/11.02 MiB
duckdb ------------------------------ 9.20 MiB/19.27 MiB
playwright ------------------------------ 7.82 MiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠦ Preparing packages... (28/33)
pyright ------------------------------ 2.12 MiB/5.31 MiB
ruff ------------------------------ 9.97 MiB/11.02 MiB
duckdb ------------------------------ 9.81 MiB/19.27 MiB
playwright ------------------------------ 8.45 MiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠦ Preparing packages... (28/33)
pyright ------------------------------ 2.14 MiB/5.31 MiB
ruff ------------------------------ 10.64 MiB/11.02 MiB
duckdb ------------------------------ 10.50 MiB/19.27 MiB
playwright ------------------------------ 9.14 MiB/43.05 MiB
Built scubaduck @ file:///workspace/scubaduck
⠦ Preparing packages... (28/33)
pyright ------------------------------ 2.14 MiB/5.31 MiB
ruff ------------------------------ 10.92 MiB/11.02 MiB
duckdb ------------------------------ 10.78 MiB/19.27 MiB
playwright ------------------------------ 9.41 MiB/43.05 MiB
⠦ Preparing packages... (28/33)
pyright ------------------------------ 2.14 MiB/5.31 MiB
duckdb ------------------------------ 10.95 MiB/19.27 MiB
playwright ------------------------------ 9.58 MiB/43.05 MiB
⠦ Preparing packages... (28/33)
pyright ------------------------------ 2.14 MiB/5.31 MiB
duckdb ------------------------------ 11.09 MiB/19.27 MiB
playwright ------------------------------ 9.70 MiB/43.05 MiB
⠧ Preparing packages... (30/33)
pyright ------------------------------ 2.17 MiB/5.31 MiB
duckdb ------------------------------ 12.29 MiB/19.27 MiB
playwright ------------------------------ 10.90 MiB/43.05 MiB
⠧ Preparing packages... (30/33)
pyright ------------------------------ 2.19 MiB/5.31 MiB
duckdb ------------------------------ 13.54 MiB/19.27 MiB
playwright ------------------------------ 12.14 MiB/43.05 MiB
⠧ Preparing packages... (30/33)
pyright ------------------------------ 2.19 MiB/5.31 MiB
duckdb ------------------------------ 14.59 MiB/19.27 MiB
playwright ------------------------------ 13.17 MiB/43.05 MiB
⠧ Preparing packages... (30/33)
pyright ------------------------------ 2.20 MiB/5.31 MiB
duckdb ------------------------------ 15.67 MiB/19.27 MiB
playwright ------------------------------ 14.25 MiB/43.05 MiB
⠇ Preparing packages... (30/33)
pyright ------------------------------ 2.22 MiB/5.31 MiB
duckdb ------------------------------ 16.89 MiB/19.27 MiB
playwright ------------------------------ 15.51 MiB/43.05 MiB
⠇ Preparing packages... (30/33)
pyright ------------------------------ 2.25 MiB/5.31 MiB
duckdb ------------------------------ 18.03 MiB/19.27 MiB
playwright ------------------------------ 16.64 MiB/43.05 MiB
⠇ Preparing packages... (30/33)
pyright ------------------------------ 2.25 MiB/5.31 MiB
duckdb ------------------------------ 19.16 MiB/19.27 MiB
playwright ------------------------------ 17.75 MiB/43.05 MiB
⠇ Preparing packages... (30/33)
pyright ------------------------------ 2.30 MiB/5.31 MiB
playwright ------------------------------ 19.41 MiB/43.05 MiB
⠇ Preparing packages... (30/33)
pyright ------------------------------ 2.31 MiB/5.31 MiB
playwright ------------------------------ 19.59 MiB/43.05 MiB
⠋ Preparing packages... (31/33)
pyright ------------------------------ 2.39 MiB/5.31 MiB
playwright ------------------------------ 21.70 MiB/43.05 MiB
⠋ Preparing packages... (31/33)
pyright ------------------------------ 2.41 MiB/5.31 MiB
playwright ------------------------------ 24.23 MiB/43.05 MiB
⠋ Preparing packages... (31/33)
pyright ------------------------------ 2.44 MiB/5.31 MiB
playwright ------------------------------ 26.62 MiB/43.05 MiB
⠋ Preparing packages... (31/33)
pyright ------------------------------ 2.45 MiB/5.31 MiB
playwright ------------------------------ 29.19 MiB/43.05 MiB
⠙ Preparing packages... (31/33)
pyright ------------------------------ 2.48 MiB/5.31 MiB
playwright ------------------------------ 31.62 MiB/43.05 MiB
⠙ Preparing packages... (31/33)
pyright ------------------------------ 2.52 MiB/5.31 MiB
playwright ------------------------------ 33.94 MiB/43.05 MiB
⠙ Preparing packages... (31/33)
pyright ------------------------------ 2.55 MiB/5.31 MiB
playwright ------------------------------ 36.27 MiB/43.05 MiB
⠙ Preparing packages... (31/33)
pyright ------------------------------ 2.58 MiB/5.31 MiB
playwright ------------------------------ 38.52 MiB/43.05 MiB
⠹ Preparing packages... (31/33)
pyright ------------------------------ 2.61 MiB/5.31 MiB
playwright ------------------------------ 40.58 MiB/43.05 MiB
⠹ Preparing packages... (31/33)
pyright ------------------------------ 2.69 MiB/5.31 MiB
playwright ------------------------------ 41.28 MiB/43.05 MiB
⠹ Preparing packages... (31/33)
pyright ------------------------------ 2.75 MiB/5.31 MiB
playwright ------------------------------ 42.22 MiB/43.05 MiB
⠹ Preparing packages... (31/33)
pyright ------------------------------ 2.78 MiB/5.31 MiB
⠸ Preparing packages... (32/33)
pyright ------------------------------ 2.83 MiB/5.31 MiB
⠸ Preparing packages... (32/33)
pyright ------------------------------ 2.97 MiB/5.31 MiB
⠸ Preparing packages... (32/33)
pyright ------------------------------ 3.08 MiB/5.31 MiB
⠸ Preparing packages... (32/33)
pyright ------------------------------ 3.20 MiB/5.31 MiB
⠼ Preparing packages... (32/33)
pyright ------------------------------ 3.29 MiB/5.31 MiB
⠼ Preparing packages... (32/33)
pyright ------------------------------ 3.48 MiB/5.31 MiB
⠼ Preparing packages... (32/33)
pyright ------------------------------ 3.67 MiB/5.31 MiB
⠼ Preparing packages... (32/33)
pyright ------------------------------ 3.79 MiB/5.31 MiB
⠼ Preparing packages... (32/33)
pyright ------------------------------ 3.92 MiB/5.31 MiB
⠴ Preparing packages... (32/33)
pyright ------------------------------ 4.22 MiB/5.31 MiB
⠴ Preparing packages... (32/33)
pyright ------------------------------ 4.39 MiB/5.31 MiB
⠴ Preparing packages... (32/33)
pyright ------------------------------ 4.52 MiB/5.31 MiB
⠴ Preparing packages... (32/33)
pyright ------------------------------ 4.59 MiB/5.31 MiB
⠦ Preparing packages... (32/33)
Prepared 33 packages in 2.81s
░░░░░░░░░░░░░░░░░░░░ [0/0] Installing wheels...
░░░░░░░░░░░░░░░░░░░░ [0/33] Installing wheels...
░░░░░░░░░░░░░░░░░░░░ [0/33] pyee==13.0.0
░░░░░░░░░░░░░░░░░░░░ [1/33] pyee==13.0.0
░░░░░░░░░░░░░░░░░░░░ [1/33] pytest-playwright==0.7.0
█░░░░░░░░░░░░░░░░░░░ [2/33] pytest-playwright==0.7.0
█░░░░░░░░░░░░░░░░░░░ [2/33] python-slugify==8.0.4
█░░░░░░░░░░░░░░░░░░░ [3/33] python-slugify==8.0.4
█░░░░░░░░░░░░░░░░░░░ [3/33] typing-extensions==4.13.2
██░░░░░░░░░░░░░░░░░░ [4/33] typing-extensions==4.13.2
██░░░░░░░░░░░░░░░░░░ [4/33] packaging==25.0
███░░░░░░░░░░░░░░░░░ [5/33] packaging==25.0
███░░░░░░░░░░░░░░░░░ [5/33] urllib3==2.4.0
███░░░░░░░░░░░░░░░░░ [6/33] urllib3==2.4.0
███░░░░░░░░░░░░░░░░░ [6/33] certifi==2025.4.26
████░░░░░░░░░░░░░░░░ [7/33] certifi==2025.4.26
████░░░░░░░░░░░░░░░░ [7/33] six==1.17.0
████░░░░░░░░░░░░░░░░ [8/33] six==1.17.0
████░░░░░░░░░░░░░░░░ [8/33] jinja2==3.1.6
█████░░░░░░░░░░░░░░░ [9/33] jinja2==3.1.6
███████████████████░ [32/33] pyright==1.1.400
████████████████████ [33/33] pyright==1.1.400
Installed 33 packages in 111ms
+ blinker==1.9.0
+ certifi==2025.4.26
+ charset-normalizer==3.4.2
+ click==8.2.0
+ duckdb==1.2.2
+ execnet==2.1.1
+ flask==3.1.1
+ greenlet==3.2.2
+ idna==3.10
+ iniconfig==2.1.0
+ itsdangerous==2.2.0
+ jinja2==3.1.6
+ markupsafe==3.0.2
+ nodeenv==1.9.1
+ packaging==25.0
+ playwright==1.52.0
+ pluggy==1.6.0
+ pyee==13.0.0
+ pyright==1.1.400
+ pytest==8.3.5
+ pytest-base-url==2.1.0
+ pytest-playwright==0.7.0
+ pytest-xdist==3.6.1
+ python-dateutil==2.9.0.post0
+ python-slugify==8.0.4
+ requests==2.32.3
+ ruff==0.11.10
+ scubaduck==0.1.0 (from file:///workspace/scubaduck)
+ six==1.17.0
+ text-unidecode==1.3
+ typing-extensions==4.13.2
+ urllib3==2.4.0
+ werkzeug==3.1.3
++ source .venv/bin/activate
+++ '[' -z '' ']'
+++ '[' -n x ']'
+++ SCRIPT_PATH=.venv/bin/activate
+++ '[' .venv/bin/activate = /tmp/NJLQjC-setup_script.sh ']'
+++ deactivate nondestructive
+++ unset -f pydoc
+++ '[' -z '' ']'
+++ '[' -z '' ']'
+++ hash -r
+++ '[' -z '' ']'
+++ unset VIRTUAL_ENV
+++ unset VIRTUAL_ENV_PROMPT
+++ '[' '!' nondestructive = nondestructive ']'
+++ VIRTUAL_ENV=/workspace/scubaduck/.venv
+++ '[' linux-gnu = cygwin ']'
+++ '[' linux-gnu = msys ']'
+++ export VIRTUAL_ENV
+++ '[' -z '' ']'
+++ unset SCRIPT_PATH
+++ _OLD_VIRTUAL_PATH=/root/.cargo/bin:/root/.rbenv/shims:/root/.rbenv/bin:/root/.rbenv/shims:/root/.local/share/swiftly/bin:/root/.bun/bin:/root/.nvm/versions/node/v22.15.1/bin:/root/.pyenv/shims:3441PYENV_ROOT/shims:/root/.pyenv/bin:/usr/local/go/bin:/root/go/bin:/root/.rbenv/bin:/root/.rbenv/shims:/root/.bun/bin:/root/.local/bin:/root/.pyenv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+++ PATH=/workspace/scubaduck/.venv/bin:/root/.cargo/bin:/root/.rbenv/shims:/root/.rbenv/bin:/root/.rbenv/shims:/root/.local/share/swiftly/bin:/root/.bun/bin:/root/.nvm/versions/node/v22.15.1/bin:/root/.pyenv/shims:3441PYENV_ROOT/shims:/root/.pyenv/bin:/usr/local/go/bin:/root/go/bin:/root/.rbenv/bin:/root/.rbenv/shims:/root/.bun/bin:/root/.local/bin:/root/.pyenv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+++ export PATH
+++ '[' xscubaduck '!=' x ']'
+++ VIRTUAL_ENV_PROMPT='(scubaduck) '
+++ export VIRTUAL_ENV_PROMPT
+++ '[' -z '' ']'
+++ '[' -z '' ']'
+++ _OLD_VIRTUAL_PS1=
+++ PS1='(scubaduck) '
+++ export PS1
+++ alias pydoc
+++ true
+++ hash -r
++ playwright install chromium
Downloading Chromium 136.0.7103.25 (playwright build v1169) from https://cdn.playwright.dev/dbazure/download/playwright/builds/chromium/1169/chromium-linux.zip
167.7 MiB [] 0% 0.0s167.7 MiB [] 0% 14.8s167.7 MiB [] 0% 11.1s167.7 MiB [] 0% 7.3s167.7 MiB [] 1% 5.0s167.7 MiB [] 1% 4.1s167.7 MiB [] 2% 4.5s167.7 MiB [] 3% 3.9s167.7 MiB [] 4% 3.0s167.7 MiB [] 5% 2.8s167.7 MiB [] 6% 2.6s167.7 MiB [] 7% 2.4s167.7 MiB [] 9% 2.2s167.7 MiB [] 10% 2.2s167.7 MiB [] 11% 2.1s167.7 MiB [] 12% 1.8s167.7 MiB [] 14% 1.8s167.7 MiB [] 15% 1.7s167.7 MiB [] 16% 1.7s167.7 MiB [] 17% 1.6s167.7 MiB [] 19% 1.5s167.7 MiB [] 20% 1.4s167.7 MiB [] 22% 1.3s167.7 MiB [] 23% 1.3s167.7 MiB [] 24% 1.3s167.7 MiB [] 26% 1.2s167.7 MiB [] 27% 1.2s167.7 MiB [] 28% 1.2s167.7 MiB [] 29% 1.1s167.7 MiB [] 30% 1.1s167.7 MiB [] 32% 1.1s167.7 MiB [] 33% 1.1s167.7 MiB [] 34% 1.0s167.7 MiB [] 36% 1.0s167.7 MiB [] 37% 1.0s167.7 MiB [] 39% 0.9s167.7 MiB [] 40% 0.9s167.7 MiB [] 41% 0.9s167.7 MiB [] 42% 0.9s167.7 MiB [] 43% 0.9s167.7 MiB [] 44% 0.9s167.7 MiB [] 46% 0.9s167.7 MiB [] 48% 0.8s167.7 MiB [] 49% 0.8s167.7 MiB [] 51% 0.7s167.7 MiB [] 52% 0.7s167.7 MiB [] 54% 0.7s167.7 MiB [] 56% 0.7s167.7 MiB [] 57% 0.6s167.7 MiB [] 58% 0.6s167.7 MiB [] 59% 0.6s167.7 MiB [] 61% 0.6s167.7 MiB [] 62% 0.6s167.7 MiB [] 63% 0.5s167.7 MiB [] 65% 0.5s167.7 MiB [] 66% 0.5s167.7 MiB [] 67% 0.5s167.7 MiB [] 69% 0.5s167.7 MiB [] 70% 0.4s167.7 MiB [] 71% 0.4s167.7 MiB [] 73% 0.4s167.7 MiB [] 74% 0.4s167.7 MiB [] 76% 0.3s167.7 MiB [] 77% 0.3s167.7 MiB [] 78% 0.3s167.7 MiB [] 79% 0.3s167.7 MiB [] 81% 0.3s167.7 MiB [] 82% 0.2s167.7 MiB [] 83% 0.2s167.7 MiB [] 85% 0.2s167.7 MiB [] 87% 0.2s167.7 MiB [] 88% 0.2s167.7 MiB [] 89% 0.1s167.7 MiB [] 90% 0.1s167.7 MiB [] 91% 0.1s167.7 MiB [] 93% 0.1s167.7 MiB [] 94% 0.1s167.7 MiB [] 95% 0.1s167.7 MiB [] 97% 0.0s167.7 MiB [] 98% 0.0s167.7 MiB [] 100% 0.0s
Chromium 136.0.7103.25 (playwright build v1169) downloaded to /root/.cache/ms-playwright/chromium-1169
Downloading FFMPEG playwright build v1011 from https://cdn.playwright.dev/dbazure/download/playwright/builds/ffmpeg/1011/ffmpeg-linux.zip
2.3 MiB [] 0% 0.0s2.3 MiB [] 5% 0.4s2.3 MiB [] 14% 0.2s2.3 MiB [] 27% 0.1s2.3 MiB [] 56% 0.1s2.3 MiB [] 100% 0.0s
FFMPEG playwright build v1011 downloaded to /root/.cache/ms-playwright/ffmpeg-1011
Downloading Chromium Headless Shell 136.0.7103.25 (playwright build v1169) from https://cdn.playwright.dev/dbazure/download/playwright/builds/chromium/1169/chromium-headless-shell-linux.zip
101.4 MiB [] 0% 0.0s101.4 MiB [] 0% 4.7s101.4 MiB [] 0% 3.2s101.4 MiB [] 2% 2.2s101.4 MiB [] 3% 1.8s101.4 MiB [] 5% 1.5s101.4 MiB [] 6% 1.3s101.4 MiB [] 8% 1.3s101.4 MiB [] 9% 1.3s101.4 MiB [] 10% 1.2s101.4 MiB [] 12% 1.1s101.4 MiB [] 13% 1.1s101.4 MiB [] 15% 1.0s101.4 MiB [] 17% 1.0s101.4 MiB [] 20% 0.9s101.4 MiB [] 22% 0.9s101.4 MiB [] 23% 0.8s101.4 MiB [] 25% 0.8s101.4 MiB [] 28% 0.7s101.4 MiB [] 30% 0.7s101.4 MiB [] 33% 0.6s101.4 MiB [] 36% 0.6s101.4 MiB [] 39% 0.5s101.4 MiB [] 42% 0.5s101.4 MiB [] 45% 0.5s101.4 MiB [] 47% 0.5s101.4 MiB [] 50% 0.4s101.4 MiB [] 53% 0.4s101.4 MiB [] 57% 0.3s101.4 MiB [] 61% 0.3s101.4 MiB [] 63% 0.3s101.4 MiB [] 64% 0.3s101.4 MiB [] 65% 0.3s101.4 MiB [] 66% 0.3s101.4 MiB [] 69% 0.3s101.4 MiB [] 71% 0.2s101.4 MiB [] 75% 0.2s101.4 MiB [] 78% 0.2s101.4 MiB [] 80% 0.2s101.4 MiB [] 82% 0.2s101.4 MiB [] 83% 0.1s101.4 MiB [] 87% 0.1s101.4 MiB [] 90% 0.1s101.4 MiB [] 93% 0.1s101.4 MiB [] 97% 0.0s101.4 MiB [] 98% 0.0s101.4 MiB [] 99% 0.0s101.4 MiB [] 100% 0.0s
Chromium Headless Shell 136.0.7103.25 (playwright build v1169) downloaded to /root/.cache/ms-playwright/chromium_headless_shell-1169
++ echo 'source .venv/bin/activate'Reading the root AGENTS.md to align with instructions.
bash: .venv/bin/activate: No such file or directory
root@3fe4fa1f8226:/workspace# ls
scubaduckroot@3fe4fa1f8226:/workspace# cd scubaduck
root@3fe4fa1f8226:/workspace/scubaduck# ls
AGENTS.md codex_setup.sh scubaduck tests
README.md pyproject.toml stubs uv.lockroot@3fe4fa1f8226:/workspace/scubaduck# cat AGENTS.md
- Use "source .venv/bin/activate" to activate the virtual environment before
running any commands
- ONLY use non-interactive commands like cat, sed, apply_patch to do edits.
Do NOT use interactive editors.
- Do NOT attempt to install packages. Only the packages specified in
pyproject.toml are available. You cannot add new packages. If you
desperately want another package, make a note of it in the final PR
description.
- Use conventional commits to format PR title
- There are no nested AGENTS.md files, this is the only agents file
- When using Playwright, ONLY use chromium browser. This browser is already
installed.
- Use "ruff check" to check lint, "ruff format" to autoformat files and
"pyright" to typecheck.
- When writing the PR description, include the original prompt supplied by the
user.
- When you add functionality to the server, add server tests to
tests/test_server.py. When you add features to the frontend, add frontend
tests to tests/test_web.pyLet's check out the repo structure and keep the instructions in mind.
root@3fe4fa1f8226:/workspace/scubaduck# ls scubaduck
__init__.py sample.csv server.py staticroot@3fe4fa1f8226:/workspace/scubaduck# ls scubaduck/static
index.html jsroot@3fe4fa1f8226:/workspace/scubaduck# ls scubaduck/static/js
chip_input.js timeseries_chart.jsroot@3fe4fa1f8226:/workspace/scubaduck# cat scubaduck/server.py
from __future__ import annotations
from dataclasses import dataclass, field, replace
from typing import Any, Dict, List, Tuple, cast
import re
from datetime import datetime, timedelta, timezone
import time
from pathlib import Path
import os
import sqlite3
import traceback
import duckdb
from dateutil import parser as dtparser
from flask import Flask, jsonify, request, send_from_directory
@dataclass
class Filter:
column: str
op: str
value: str | int | float | list[str] | None
@dataclass
class QueryParams:
start: str | None = None
end: str | None = None
order_by: str | None = None
order_dir: str = "ASC"
limit: int | None = None
columns: list[str] = field(default_factory=lambda: [])
filters: list[Filter] = field(default_factory=lambda: [])
derived_columns: dict[str, str] = field(default_factory=lambda: {})
graph_type: str = "samples"
group_by: list[str] = field(default_factory=lambda: [])
aggregate: str | None = None
show_hits: bool = False
x_axis: str | None = None
granularity: str = "Auto"
fill: str = "0"
def _load_database(path: Path) -> duckdb.DuckDBPyConnection:
if not path.exists():
raise FileNotFoundError(path)
ext = path.suffix.lower()
if ext == ".csv":
con = duckdb.connect()
con.execute(
f"CREATE TABLE events AS SELECT * FROM read_csv_auto('{path.as_posix
()}')"
)
elif ext in {".db", ".sqlite"}:
con = duckdb.connect()
sconn = sqlite3.connect(path)
info = sconn.execute("PRAGMA table_info(events)").fetchall()
col_defs = ", ".join(f"{r[1]} {r[2]}" for r in info)
con.execute(f"CREATE TABLE events ({col_defs})")
placeholders = ",".join("?" for _ in info)
for row in sconn.execute("SELECT * FROM events"):
con.execute(f"INSERT INTO events VALUES ({placeholders})", row)
sconn.close()
else:
con = duckdb.connect(path)
return con
_REL_RE = re.compile(
r"([+-]?\d+(?:\.\d*)?)\s*(hour|hours|day|days|week|weeks|fortnight|fortnight
s)",
re.IGNORECASE,
)
def parse_time(val: str | None) -> str | None:
"""Parse an absolute or relative time string into ``YYYY-MM-DD HH:MM:SS``.""
"
if val is None or val == "":
return None
s = val.strip()
if s.lower() == "now":
dt = datetime.now(timezone.utc)
return dt.replace(microsecond=0).strftime("%Y-%m-%d %H:%M:%S")
m = _REL_RE.fullmatch(s)
if m:
qty = float(m.group(1))
unit = m.group(2).lower()
delta: timedelta
if unit.startswith("hour"):
delta = timedelta(hours=qty)
elif unit.startswith("day"):
delta = timedelta(days=qty)
elif unit.startswith("week"):
delta = timedelta(weeks=qty)
elif unit.startswith("fortnight"):
delta = timedelta(weeks=2 * qty)
else: # pragma: no cover - defensive
raise ValueError(f"Unsupported unit: {unit}")
dt = datetime.now(timezone.utc) + delta
return dt.replace(microsecond=0).strftime("%Y-%m-%d %H:%M:%S")
dt = dtparser.parse(s)
return dt.replace(microsecond=0, tzinfo=None).strftime("%Y-%m-%d %H:%M:%S")
def _granularity_seconds(granularity: str, start: str | None, end: str | None) -
> int:
gran = granularity.lower()
mapping = {
"1 second": 1,
"5 seconds": 5,
"10 seconds": 10,
"30 seconds": 30,
"1 minute": 60,
"4 minutes": 240,
"5 minutes": 300,
"10 minutes": 600,
"15 minutes": 900,
...
break
if params.x_axis is None or params.x_axis not in valid_cols:
return jsonify({"error": "Invalid x_axis"}), 400
ctype = column_types.get(params.x_axis, "").upper()
if not any(t in ctype for t in ["TIMESTAMP", "DATE", "TIME"]):
return jsonify({"error": "x_axis must be a time column"}), 400
for col in params.columns:
if col not in valid_cols:
return jsonify({"error": f"Unknown column: {col}"}), 400
for col in params.group_by:
if col not in valid_cols:
return jsonify({"error": f"Unknown column: {col}"}), 400
if params.order_by and params.order_by not in valid_cols:
return jsonify({"error": f"Unknown column: {params.order_by}"}), 400
if params.group_by or params.graph_type == "timeseries":
agg = (params.aggregate or "avg").lower()
if agg.startswith("p") or agg == "sum":
need_numeric = True
allow_time = False
elif agg == "avg" or agg in {"min", "max"}:
need_numeric = False
allow_time = True
else:
need_numeric = False
allow_time = False
if need_numeric or allow_time:
for c in params.columns:
if c in params.group_by or c == params.x_axis:
continue
if c not in column_types:
continue
ctype = column_types.get(c, "").upper()
is_numeric = any(
t in ctype
for t in [
"INT",
"DECIMAL",
"REAL",
"DOUBLE",
"FLOAT",
"NUMERIC",
"HUGEINT",
]
)
is_time = "TIMESTAMP" in ctype or "DATE" in ctype or "TIME"
in ctype
if need_numeric and not is_numeric:
return (
jsonify(
{
"error": f"Aggregate {agg} cannot be applied
to column {c}",
}
),
400,
)
if allow_time and not (is_numeric or is_time):
return (
jsonify(
{
"error": f"Aggregate {agg} cannot be applied
to column {c}",
}
),
400,
)
if params.start is None or params.end is None:
axis = params.x_axis or "timestamp"
row = cast(
tuple[datetime | None, datetime | None],
con.execute(f"SELECT min({axis}), max({axis}) FROM events").fetc
hall()[
0
],
)
mn, mx = row
if params.start is None and mn is not None:
params.start = (
mn.strftime("%Y-%m-%d %H:%M:%S") if not isinstance(mn, str)
else mn
)
if params.end is None and mx is not None:
params.end = (
mx.strftime("%Y-%m-%d %H:%M:%S") if not isinstance(mx, str)
else mx
)
bucket_size: int | None = None
if params.graph_type == "timeseries":
bucket_size = _granularity_seconds(
params.granularity,
params.start if isinstance(params.start, str) else None,
params.end if isinstance(params.end, str) else None,
)
sql = build_query(params, column_types)
try:
rows = con.execute(sql).fetchall()
except Exception as exc:
tb = traceback.format_exc()
print(f"Query failed:\n{sql}\n{tb}")
return (
jsonify({"sql": sql, "error": str(exc), "traceback": tb}),
400,
)
result: Dict[str, Any] = {"sql": sql, "rows": rows}
if params.start is not None:
result["start"] = str(params.start)
if params.end is not None:
result["end"] = str(params.end)
if bucket_size is not None:
result["bucket_size"] = bucket_size
return jsonify(result)
return app
app = create_app()
if __name__ == "__main__":
app.run(debug=True)root@3fe4fa1f8226:/workspace/scubaduck# sed -n '1,160p' scubaduck/server.py
from __future__ import annotations
from dataclasses import dataclass, field, replace
from typing import Any, Dict, List, Tuple, cast
import re
from datetime import datetime, timedelta, timezone
import time
from pathlib import Path
import os
import sqlite3
import traceback
import duckdb
from dateutil import parser as dtparser
from flask import Flask, jsonify, request, send_from_directory
@dataclass
class Filter:
column: str
op: str
value: str | int | float | list[str] | None
@dataclass
class QueryParams:
start: str | None = None
end: str | None = None
order_by: str | None = None
order_dir: str = "ASC"
limit: int | None = None
columns: list[str] = field(default_factory=lambda: [])
filters: list[Filter] = field(default_factory=lambda: [])
derived_columns: dict[str, str] = field(default_factory=lambda: {})
graph_type: str = "samples"
group_by: list[str] = field(default_factory=lambda: [])
aggregate: str | None = None
show_hits: bool = False
x_axis: str | None = None
granularity: str = "Auto"
fill: str = "0"
def _load_database(path: Path) -> duckdb.DuckDBPyConnection:
if not path.exists():
raise FileNotFoundError(path)
ext = path.suffix.lower()
if ext == ".csv":
con = duckdb.connect()
con.execute(
f"CREATE TABLE events AS SELECT * FROM read_csv_auto('{path.as_posix
()}')"
)
elif ext in {".db", ".sqlite"}:
con = duckdb.connect()
sconn = sqlite3.connect(path)
info = sconn.execute("PRAGMA table_info(events)").fetchall()
col_defs = ", ".join(f"{r[1]} {r[2]}" for r in info)
con.execute(f"CREATE TABLE events ({col_defs})")
placeholders = ",".join("?" for _ in info)
for row in sconn.execute("SELECT * FROM events"):
con.execute(f"INSERT INTO events VALUES ({placeholders})", row)
sconn.close()
else:
con = duckdb.connect(path)
return con
_REL_RE = re.compile(
r"([+-]?\d+(?:\.\d*)?)\s*(hour|hours|day|days|week|weeks|fortnight|fortnight
s)",
re.IGNORECASE,
)
def parse_time(val: str | None) -> str | None:
"""Parse an absolute or relative time string into ``YYYY-MM-DD HH:MM:SS``.""
"
if val is None or val == "":
return None
s = val.strip()
if s.lower() == "now":
dt = datetime.now(timezone.utc)
return dt.replace(microsecond=0).strftime("%Y-%m-%d %H:%M:%S")
m = _REL_RE.fullmatch(s)
if m:
qty = float(m.group(1))
unit = m.group(2).lower()
delta: timedelta
if unit.startswith("hour"):
delta = timedelta(hours=qty)
elif unit.startswith("day"):
delta = timedelta(days=qty)
elif unit.startswith("week"):
delta = timedelta(weeks=qty)
elif unit.startswith("fortnight"):
delta = timedelta(weeks=2 * qty)
else: # pragma: no cover - defensive
raise ValueError(f"Unsupported unit: {unit}")
dt = datetime.now(timezone.utc) + delta
return dt.replace(microsecond=0).strftime("%Y-%m-%d %H:%M:%S")
dt = dtparser.parse(s)
return dt.replace(microsecond=0, tzinfo=None).strftime("%Y-%m-%d %H:%M:%S")
def _granularity_seconds(granularity: str, start: str | None, end: str | None) -
> int:
gran = granularity.lower()
mapping = {
"1 second": 1,
"5 seconds": 5,
"10 seconds": 10,
"30 seconds": 30,
"1 minute": 60,
"4 minutes": 240,
"5 minutes": 300,
"10 minutes": 600,
"15 minutes": 900,
"30 minutes": 1800,
"1 hour": 3600,
"3 hours": 10800,
"6 hours": 21600,
"1 day": 86400,
"1 week": 604800,
"30 days": 2592000,
}
if gran in mapping:
return mapping[gran]
if gran in {"auto", "fine"} and start and end:
try:
s = dtparser.parse(start)
e = dtparser.parse(end)
except Exception:
return 3600
total = max((e - s).total_seconds(), 1)
buckets = 100 if gran == "auto" else 500
return max(int(total // buckets), 1)
return 3600
def build_query(params: QueryParams, column_types: Dict[str, str] | None = None)
-> str:
select_parts: list[str] = []
group_cols = params.group_by[:]
if params.graph_type == "timeseries":
sec = _granularity_seconds(params.granularity, params.start, params.end)
x_axis = params.x_axis or "timestamp"
bucket_expr = (
f"TIMESTAMP 'epoch' + INTERVAL '{sec} second' * "
f"CAST(floor(epoch({x_axis})/{sec}) AS BIGINT)"
)
select_parts.append(f"{bucket_expr} AS bucket")
group_cols = ["bucket"] + group_cols
has_agg = bool(group_cols) or params.aggregate is not None
if has_agg:
select_cols = (
group_cols[1:] if params.graph_type == "timeseries" else group_cols
)
select_parts.extend(select_cols)
agg = (params.aggregate or "avg").lower()I’ll open the remaining files to look for any functions.
root@3fe4fa1f8226:/workspace/scubaduck# grep -n "def create_app" -n scubaduck/se
rver.py
254:def create_app(db_file: str | Path | None = None) -> Flask:root@3fe4fa1f8226:/workspace/scubaduck# sed -n '254,500p' scubaduck/server.py
def create_app(db_file: str | Path | None = None) -> Flask:
app = Flask(__name__, static_folder="static")
if db_file is None:
env_db = os.environ.get("SCUBADUCK_DB")
if env_db:
db_file = env_db
db_path = Path(db_file or Path(__file__).with_name("sample.csv")).resolve()
con = _load_database(db_path)
column_types: Dict[str, str] = {
r[1]: r[2] for r in con.execute("PRAGMA table_info(events)").fetchall()
}
sample_cache: Dict[Tuple[str, str], Tuple[List[str], float]] = {}
CACHE_TTL = 60.0
CACHE_LIMIT = 200
@app.route("/")
def index() -> Any: # pyright: ignore[reportUnusedFunction]
assert app.static_folder is not None
return send_from_directory(app.static_folder, "index.html")
@app.route("/js/<path:filename>")
def js(filename: str) -> Any: # pyright: ignore[reportUnusedFunction]
assert app.static_folder is not None
folder = Path(app.static_folder) / "js"
return send_from_directory(folder, filename)
@app.route("/api/columns")
def columns() -> Any: # pyright: ignore[reportUnusedFunction]
rows = con.execute("PRAGMA table_info(events)").fetchall()
return jsonify([{"name": r[1], "type": r[2]} for r in rows])
def _cache_get(key: Tuple[str, str]) -> List[str] | None:
item = sample_cache.get(key)
if item is None:
return None
vals, ts = item
if time.time() - ts > CACHE_TTL:
del sample_cache[key]
return None
sample_cache[key] = (vals, time.time())
return vals
def _cache_set(key: Tuple[str, str], vals: List[str]) -> None:
sample_cache[key] = (vals, time.time())
if len(sample_cache) > CACHE_LIMIT:
oldest = min(sample_cache.items(), key=lambda kv: kv[1][1])[0]
del sample_cache[oldest]
@app.route("/api/samples")
def sample_values() -> Any: # pyright: ignore[reportUnusedFunction]
column = request.args.get("column")
substr = request.args.get("q", "")
if not column or column not in column_types:
return jsonify([])
ctype = column_types[column].upper()
if "CHAR" not in ctype and "STRING" not in ctype and "VARCHAR" not in ct
ype:
return jsonify([])
key = (column, substr)
cached = _cache_get(key)
if cached is not None:
return jsonify(cached)
rows = con.execute(
f"SELECT DISTINCT {column} FROM events WHERE CAST({column} AS VARCHA
R) ILIKE '%' || ? || '%' LIMIT 20",
[substr],
).fetchall()
values = [r[0] for r in rows]
_cache_set(key, values)
return jsonify(values)
@app.route("/api/query", methods=["POST"])
def query() -> Any: # pyright: ignore[reportUnusedFunction]
payload = request.get_json(force=True)
try:
start = parse_time(payload.get("start"))
end = parse_time(payload.get("end"))
except Exception as exc:
return jsonify({"error": str(exc)}), 400
params = QueryParams(
start=start,
end=end,
order_by=payload.get("order_by"),
order_dir=payload.get("order_dir", "ASC"),
limit=payload.get("limit"),
columns=payload.get("columns", []),
derived_columns=payload.get("derived_columns", {}),
graph_type=payload.get("graph_type", "samples"),
group_by=payload.get("group_by", []),
aggregate=payload.get("aggregate"),
show_hits=payload.get("show_hits", False),
x_axis=payload.get("x_axis"),
granularity=payload.get("granularity", "Auto"),
fill=payload.get("fill", "0"),
)
for f in payload.get("filters", []):
params.filters.append(Filter(f["column"], f["op"], f.get("value")))
if params.graph_type not in {"table", "timeseries"} and (
params.group_by or params.aggregate or params.show_hits
):
return (
jsonify(
{
"error": "group_by, aggregate and show_hits are only val
id for table or timeseries view"
}
),
400,
)
valid_cols = set(column_types.keys())
valid_cols.update(params.derived_columns.keys())
if params.graph_type == "timeseries":
if params.x_axis is None:
for cand in ["time", "timestamp"]:
if cand in valid_cols:
params.x_axis = cand
break
if params.x_axis is None or params.x_axis not in valid_cols:
return jsonify({"error": "Invalid x_axis"}), 400
ctype = column_types.get(params.x_axis, "").upper()
if not any(t in ctype for t in ["TIMESTAMP", "DATE", "TIME"]):
return jsonify({"error": "x_axis must be a time column"}), 400
for col in params.columns:
if col not in valid_cols:
return jsonify({"error": f"Unknown column: {col}"}), 400
for col in params.group_by:
if col not in valid_cols:
return jsonify({"error": f"Unknown column: {col}"}), 400
if params.order_by and params.order_by not in valid_cols:
return jsonify({"error": f"Unknown column: {params.order_by}"}), 400
if params.group_by or params.graph_type == "timeseries":
agg = (params.aggregate or "avg").lower()
if agg.startswith("p") or agg == "sum":
need_numeric = True
allow_time = False
elif agg == "avg" or agg in {"min", "max"}:
need_numeric = False
allow_time = True
else:
need_numeric = False
allow_time = False
if need_numeric or allow_time:
for c in params.columns:
if c in params.group_by or c == params.x_axis:
continue
if c not in column_types:
continue
ctype = column_types.get(c, "").upper()
is_numeric = any(
t in ctype
for t in [
"INT",
"DECIMAL",
"REAL",
"DOUBLE",
"FLOAT",
"NUMERIC",
"HUGEINT",
]
)
is_time = "TIMESTAMP" in ctype or "DATE" in ctype or "TIME"
in ctype
if need_numeric and not is_numeric:
return (
jsonify(
{
"error": f"Aggregate {agg} cannot be applied
to column {c}",
}
),
400,
)
if allow_time and not (is_numeric or is_time):
return (
jsonify(
{
"error": f"Aggregate {agg} cannot be applied
to column {c}",
}
),
400,
)
if params.start is None or params.end is None:
axis = params.x_axis or "timestamp"
row = cast(
tuple[datetime | None, datetime | None],
con.execute(f"SELECT min({axis}), max({axis}) FROM events").fetc
hall()[
0
],
)
mn, mx = row
if params.start is None and mn is not None:
params.start = (
mn.strftime("%Y-%m-%d %H:%M:%S") if not isinstance(mn, str)
else mn
)
if params.end is None and mx is not None:
params.end = (
mx.strftime("%Y-%m-%d %H:%M:%S") if not isinstance(mx, str)
else mx
)
bucket_size: int | None = None
if params.graph_type == "timeseries":
bucket_size = _granularity_seconds(
params.granularity,
params.start if isinstance(params.start, str) else None,
params.end if isinstance(params.end, str) else None,
)
sql = build_query(params, column_types)
try:
rows = con.execute(sql).fetchall()
except Exception as exc:
tb = traceback.format_exc()
print(f"Query failed:\n{sql}\n{tb}")
return (
jsonify({"sql": sql, "error": str(exc), "traceback": tb}),
400,
)
result: Dict[str, Any] = {"sql": sql, "rows": rows}
if params.start is not None:
result["start"] = str(params.start)
if params.end is not None:
result["end"] = str(params.end)
if bucket_size is not None:
result["bucket_size"] = bucket_size
return jsonify(result)
return app
app = create_app()
if __name__ == "__main__":
app.run(debug=True)I'll update to use DuckDB's sqlite extension for direct access and add table querying.
"limit": 10,
"columns": ["timestamp", "event", "value", "user"],
"filters": [],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert data
rows = data["rows"]
# We expect first three rows (until 2024-01-02 00:00:00)
assert len(rows) == 3
assert rows[0][1] == "login"
assert rows[1][1] == "logout"
def test_js_served() -> None:
app = server.app
client = app.test_client()
rv = client.get("/js/chip_input.js")
assert rv.status_code == 200
assert b"initChipInput" in rv.data
def test_filter_multi_token() -> None:
app = server.app
client = app.test_client()
payload = {
"start": "2024-01-01 00:00:00",
"end": "2024-01-02 03:00:00",
"order_by": "timestamp",
"limit": 10,
"columns": ["timestamp", "event", "value", "user"],
"filters": [{"column": "user", "op": "=", "value": ["alice", "charlie"]}
],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert data
rows = data["rows"]
# Should only return rows for alice and charlie
assert len(rows) == 3
assert rows[0][3] == "alice"
assert rows[-1][3] == "charlie"
def test_empty_filter_is_noop() -> None:
app = server.app
client = app.test_client()
base_payload = {
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"limit": 100,
"columns": ["timestamp", "event", "value", "user"],
}
no_filter = {**base_payload, "filters": []}
empty_filter = {
**base_payload,
"filters": [{"column": "user", "op": "=", "value": None}],
}
rv1 = client.post(
"/api/query", data=json.dumps(no_filter), content_type="application/json
"
)
rv2 = client.post(
"/api/query", data=json.dumps(empty_filter), content_type="application/j
son"
)
rows1 = rv1.get_json()["rows"]
rows2 = rv2.get_json()["rows"]
assert rows1 == rows2
def test_select_columns() -> None:
app = server.app
client = app.test_client()
payload = {
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"order_by": "timestamp",
"limit": 10,
"columns": ["timestamp", "user"],
"filters": [],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert data
rows = data["rows"]
assert len(rows[0]) == 2
assert rows[0][1] == "alice"
def test_string_filter_ops() -> None:
app = server.app
client = app.test_client()
base = {
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"order_by": "timestamp",
"limit": 100,
"columns": ["timestamp", "event", "value", "user"],
}
contains = {
**base,
"filters": [{"column": "user", "op": "contains", "value": "ali"}],
}
rv = client.post(
"/api/query", data=json.dumps(contains), content_type="application/json"
)
rows = rv.get_json()["rows"]
assert all("ali" in r[3] for r in rows)
regex = {
**base,
"filters": [{"column": "user", "op": "~", "value": "^a.*"}],
}
rv = client.post(
"/api/query", data=json.dumps(regex), content_type="application/json"
...
assert ts == parser.parse("2024-01-01 12:00:00")
def test_timeseries_basic() -> None:
app = server.app
client = app.test_client()
payload = {
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "timeseries",
"limit": 100,
"group_by": ["user"],
"aggregate": "Count",
"columns": ["value"],
"x_axis": "timestamp",
"granularity": "1 day",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert len(data["rows"]) == 4
def test_timeseries_orders_by_xaxis() -> None:
app = server.app
client = app.test_client()
payload = {
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "timeseries",
"limit": 100,
"columns": ["value"],
"x_axis": "timestamp",
"granularity": "1 day",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
rows = data["rows"]
from dateutil import parser
timestamps = [parser.parse(r[0]).replace(tzinfo=None) for r in rows]
assert timestamps == sorted(timestamps)
def test_timeseries_string_column_error() -> None:
app = server.app
client = app.test_client()
payload = {
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "timeseries",
"limit": 7,
"columns": ["timestamp", "event", "value", "user"],
"x_axis": "timestamp",
"granularity": "1 hour",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 400
assert "Aggregate" in data["error"]
def test_derived_column_basic() -> None:
app = server.app
client = app.test_client()
payload = {
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"limit": 10,
"columns": ["timestamp"],
"derived_columns": {"val2": "value * 2"},
"filters": [],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert data["rows"][0][1] == 20
def test_timeseries_derived_column() -> None:
app = server.app
client = app.test_client()
payload = {
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "timeseries",
"granularity": "1 hour",
"limit": 7,
"columns": ["value"],
"derived_columns": {"derived_1": "value * 2"},
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
rows = data["rows"]
assert all(r[2] == r[1] * 2 for r in rows)
def test_default_start_end_returned() -> None:
app = server.app
client = app.test_client()
payload = {
"order_by": "timestamp",
"limit": 5,
"columns": ["timestamp"],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert data["start"] == "2024-01-01 00:00:00"
assert data["end"] == "2024-01-02 03:00:00"from pathlib import Path
import duckdb
from scubaduck import server
import pytest
def test_basic_query() -> None:
app = server.app
client = app.test_client()
payload = {
"start": "2024-01-01 00:00:00",
"end": "2024-01-02 00:00:00",
"order_by": "timestamp",
"order_dir": "ASC",
"limit": 10,
"columns": ["timestamp", "event", "value", "user"],
"filters": [],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert data
rows = data["rows"]
# We expect first three rows (until 2024-01-02 00:00:00)
assert len(rows) == 3
assert rows[0][1] == "login"
assert rows[1][1] == "logout"
def test_js_served() -> None:
app = server.app
client = app.test_client()
rv = client.get("/js/chip_input.js")
assert rv.status_code == 200
assert b"initChipInput" in rv.data
def test_filter_multi_token() -> None:
app = server.app
client = app.test_client()
payload = {
"start": "2024-01-01 00:00:00",
"end": "2024-01-02 03:00:00",
"order_by": "timestamp",
"limit": 10,
"columns": ["timestamp", "event", "value", "user"],
"filters": [{"column": "user", "op": "=", "value": ["alice", "charlie"]}
],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert data
rows = data["rows"]
# Should only return rows for alice and charlie
assert len(rows) == 3
assert rows[0][3] == "alice"
assert rows[-1][3] == "charlie"
def test_empty_filter_is_noop() -> None:
app = server.app
client = app.test_client()
base_payload = {
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"limit": 100,
"columns": ["timestamp", "event", "value", "user"],
}
no_filter = {**base_payload, "filters": []}
empty_filter = {
**base_payload,
"filters": [{"column": "user", "op": "=", "value": None}],
}I'll keep scrolling to check the remaining part of the file.
"/api/query", data=json.dumps(no_filter), content_type="application/json
"
)
rv2 = client.post(
"/api/query", data=json.dumps(empty_filter), content_type="application/j
son"
)
rows1 = rv1.get_json()["rows"]
rows2 = rv2.get_json()["rows"]
assert rows1 == rows2
def test_select_columns() -> None:
app = server.app
client = app.test_client()
payload = {
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"order_by": "timestamp",
"limit": 10,
"columns": ["timestamp", "user"],
"filters": [],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert data
rows = data["rows"]
assert len(rows[0]) == 2
assert rows[0][1] == "alice"
def test_string_filter_ops() -> None:
app = server.app
client = app.test_client()
base = {
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"order_by": "timestamp",
"limit": 100,
"columns": ["timestamp", "event", "value", "user"],
}
contains = {
**base,
"filters": [{"column": "user", "op": "contains", "value": "ali"}],
}
rv = client.post(
"/api/query", data=json.dumps(contains), content_type="application/json"
)
rows = rv.get_json()["rows"]
assert all("ali" in r[3] for r in rows)
regex = {
**base,
"filters": [{"column": "user", "op": "~", "value": "^a.*"}],
}
rv = client.post(
"/api/query", data=json.dumps(regex), content_type="application/json"
)
rows = rv.get_json()["rows"]
assert all(r[3].startswith("a") for r in rows)
assert len(rows) == 2
not_empty = {**base, "filters": [{"column": "user", "op": "!empty"}]}
rv = client.post(
"/api/query", data=json.dumps(not_empty), content_type="application/json
"
)
assert len(rv.get_json()["rows"]) == 4
def _make_payload() -> dict[str, object]:
return {
"start": "2024-01-01 00:00:00",
"end": "2024-01-02 00:00:00",
"order_by": "timestamp",
"order_dir": "ASC",
"limit": 10,
"columns": ["timestamp", "event", "value", "user"],
"filters": [],sqlite_file = tmp_path / "events.sqlite"
import sqlite3
conn = sqlite3.connect(sqlite_file)
conn.execute(
"CREATE TABLE events (timestamp TEXT, event TEXT, value INTEGER, user TE
XT)"
)
with open(csv_file) as f:
next(f)
for line in f:
ts, ev, val, user = line.strip().split(",")
conn.execute(
"INSERT INTO events VALUES (?, ?, ?, ?)", (ts, ev, int(val), use
r)
)
conn.commit()
conn.close() # pyright: ignore[reportUnknownMemberType, reportAttributeAcce
ssIssue]
duckdb_file = tmp_path / "events.duckdb"
con = duckdb.connect(duckdb_file)
con.execute(
f"CREATE TABLE events AS SELECT * FROM read_csv_auto('{csv_file.as_posix
()}')"
)
con.close() # pyright: ignore[reportUnknownMemberType, reportAttributeAcces
sIssue]
for db in (csv_file, sqlite_file, duckdb_file):
app = server.create_app(db)
client = app.test_client()
payload = _make_payload()
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/js
on"
)
rows = rv.get_json()["rows"]
assert len(rows) == 3
def test_envvar_db(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
csv_file = tmp_path / "custom.csv"
csv_file.write_text("timestamp,event,value,user\n2024-01-01 00:00:00,login,5
,bob\n")
monkeypatch.setenv("SCUBADUCK_DB", str(csv_file))
app = server.create_app()
client = app.test_client()
payload = _make_payload()
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
rows = rv.get_json()["rows"]
assert len(rows) == 1
def test_envvar_db_missing(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> N
one:
missing = tmp_path / "missing.sqlite"
monkeypatch.setenv("SCUBADUCK_DB", str(missing))
with pytest.raises(FileNotFoundError):
server.create_app()
def test_group_by_table() -> None:
app = server.app
client = app.test_client()
payload = {
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "table",
"order_by": "user",
"limit": 10,
"columns": ["value"],
"group_by": ["user"],
"aggregate": "Sum",
"show_hits": True,
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
rows = rv.get_json()["rows"]
assert rows[0][0] == "alice"
assert rows[0][1] == 2
assert rows[0][2] == 40
def test_relative_time_query(monkeypatch: pytest.MonkeyPatch) -> None:
app = server.app
client = app.test_client()
from datetime import datetime
fixed_now = datetime(2024, 1, 2, 4, 0, 0)
class FixedDateTime(datetime):
@classmethod
def now(cls, tz=None): # type: ignore[override]
return fixed_now if tz is None else fixed_now.astimezone(tz)
monkeypatch.setattr(server, "datetime", FixedDateTime)
payload = {
"start": "-1 hour",
"end": "now",
"order_by": "timestamp",
"limit": 100,
"columns": ["timestamp", "event", "value", "user"],
"filters": [],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert len(data["rows"]) == 1
assert data["rows"][0][3] == "charlie"
def test_invalid_time_error() -> None:
app = server.app
client = app.test_client()
payload = {
"start": "nonsense",
"end": "now",
"order_by": "timestamp",
"limit": 10,
"columns": ["timestamp"],
"filters": [],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 400
assert "error" in data
def test_query_error_returns_sql_and_traceback() -> None:
app = server.app
client = app.test_client()
payload = {
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"columns": ["event"],
"group_by": ["user"],
"aggregate": "avg",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 400
assert "error" in data
def test_table_unknown_column_error() -> None:
app = server.app
client = app.test_client()
payload = {
"start": "2024-01-01 00:00:00","aggregate": "Count",
"show_hits": True,
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 400
assert "Unknown column" in data["error"]
def test_samples_view_rejects_group_by() -> None:
app = server.app
client = app.test_client()
payload = {
"graph_type": "samples",
"group_by": ["user"],
"columns": ["timestamp"],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 400
assert "only valid" in data["error"]
def test_table_avg_with_timestamp() -> None:
app = server.app
client = app.test_client()
payload = {
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "table",
"order_by": "user",
"limit": 100,
"columns": ["user", "timestamp", "value"],
"group_by": ["user"],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert "error" not in data
rows = data["rows"]
assert rows[0][0] == "alice"
from dateutil import parser
ts = parser.parse(rows[0][1]).replace(tzinfo=None)
assert ts == parser.parse("2024-01-01 12:00:00")
def test_timeseries_basic() -> None:
app = server.app
client = app.test_client()
payload = {
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "timeseries",
"limit": 100,
"group_by": ["user"],
"aggregate": "Count",
"columns": ["value"],
"x_axis": "timestamp",
"granularity": "1 day",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert len(data["rows"]) == 4
def test_timeseries_orders_by_xaxis() -> None:
app = server.app
client = app.test_client()
payload = {
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "timeseries",
"limit": 100,
"columns": ["value"],
"x_axis": "timestamp",
"granularity": "1 day",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
rows = data["rows"]
from dateutil import parser
timestamps = [parser.parse(r[0]).replace(tzinfo=None) for r in rows]
assert timestamps == sorted(timestamps)
def test_timeseries_string_column_error() -> None:
app = server.app
client = app.test_client()
payload = {
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "timeseries",
"limit": 7,
"columns": ["timestamp", "event", "value", "user"],
"x_axis": "timestamp",
"granularity": "1 hour",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 400
assert "Aggregate" in data["error"]
def test_derived_column_basic() -> None:
app = server.app
client = app.test_client()
payload = {
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"limit": 10,
"columns": ["timestamp"],
"derived_columns": {"val2": "value * 2"},
"filters": [],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert data["rows"][0][1] == 20
def test_timeseries_derived_column() -> None:
app = server.app
client = app.test_client()
payload = {
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "timeseries",
"granularity": "1 hour",
"limit": 7,
"columns": ["value"],
"derived_columns": {"derived_1": "value * 2"},
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
rows = data["rows"]
assert all(r[2] == r[1] * 2 for r in rows)
def test_default_start_end_returned() -> None:
app = server.app
client = app.test_client()
payload = {
"order_by": "timestamp",
"limit": 5,
"columns": ["timestamp"],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert data["start"] == "2024-01-01 00:00:00"
assert data["end"] == "2024-01-02 03:00:00"root@3fe4fa1f8226:/workspace/scubaduck# grep -n "_make_payload" -n tests/test_se
rver.py
152:def _make_payload() -> dict[str, object]:
195: payload = _make_payload()
209: payload = _make_payload()I'm checking from line 152 to the end, making sure I catch all tests.
"end": "2024-01-02 00:00:00",
"order_by": "timestamp",
"order_dir": "ASC",
"limit": 10,
"columns": ["timestamp", "event", "value", "user"],
"filters": [],
}
def test_database_types(tmp_path: Path) -> None:
csv_file = tmp_path / "events.csv"
csv_file.write_text(Path("scubaduck/sample.csv").read_text())
sqlite_file = tmp_path / "events.sqlite"
import sqlite3
conn = sqlite3.connect(sqlite_file)
conn.execute(
"CREATE TABLE events (timestamp TEXT, event TEXT, value INTEGER, user TE
XT)"
)
with open(csv_file) as f:
next(f)
for line in f:
ts, ev, val, user = line.strip().split(",")
conn.execute(
"INSERT INTO events VALUES (?, ?, ?, ?)", (ts, ev, int(val), use
r)
)
conn.commit()
conn.close() # pyright: ignore[reportUnknownMemberType, reportAttributeAcce
ssIssue]
duckdb_file = tmp_path / "events.duckdb"
con = duckdb.connect(duckdb_file)
con.execute(
f"CREATE TABLE events AS SELECT * FROM read_csv_auto('{csv_file.as_posix
()}')"
)
con.close() # pyright: ignore[reportUnknownMemberType, reportAttributeAcces
sIssue]
for db in (csv_file, sqlite_file, duckdb_file):
app = server.create_app(db)
client = app.test_client()
payload = _make_payload()
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/js
on"
)
rows = rv.get_json()["rows"]
assert len(rows) == 3
def test_envvar_db(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
csv_file = tmp_path / "custom.csv"
csv_file.write_text("timestamp,event,value,user\n2024-01-01 00:00:00,login,5
,bob\n")
monkeypatch.setenv("SCUBADUCK_DB", str(csv_file))
app = server.create_app()
client = app.test_client()
payload = _make_payload()
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
rows = rv.get_json()["rows"]
assert len(rows) == 1
def test_envvar_db_missing(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> N
one:
missing = tmp_path / "missing.sqlite"
monkeypatch.setenv("SCUBADUCK_DB", str(missing))
with pytest.raises(FileNotFoundError):
server.create_app()
def test_group_by_table() -> None:
app = server.app
client = app.test_client()
payload = {
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "table",
"order_by": "user",
"limit": 10,
"columns": ["value"],
"group_by": ["user"],
"aggregate": "Sum",
"show_hits": True,
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)I spotted truncation at 152-240, but there’s more data past 240.
assert rows[0][1] == 2
assert rows[0][2] == 40
def test_relative_time_query(monkeypatch: pytest.MonkeyPatch) -> None:
app = server.app
client = app.test_client()
from datetime import datetime
fixed_now = datetime(2024, 1, 2, 4, 0, 0)
class FixedDateTime(datetime):
@classmethod
def now(cls, tz=None): # type: ignore[override]
return fixed_now if tz is None else fixed_now.astimezone(tz)
monkeypatch.setattr(server, "datetime", FixedDateTime)
payload = {
"start": "-1 hour",
"end": "now",
"order_by": "timestamp",
"limit": 100,
"columns": ["timestamp", "event", "value", "user"],
"filters": [],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert len(data["rows"]) == 1
assert data["rows"][0][3] == "charlie"
def test_invalid_time_error() -> None:
app = server.app
client = app.test_client()
payload = {
"start": "nonsense",
"end": "now",
"order_by": "timestamp",
"limit": 10,
"columns": ["timestamp"],
"filters": [],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 400
assert "error" in data
def test_query_error_returns_sql_and_traceback() -> None:
app = server.app
client = app.test_client()
payload = {
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"columns": ["event"],
"group_by": ["user"],
"aggregate": "avg",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 400
assert "error" in data
def test_table_unknown_column_error() -> None:
app = server.app
client = app.test_client()
payload = {
"start": "2024-01-01 00:00:00","group_by": ["user"],
"aggregate": "Count",
"show_hits": True,
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 400
assert "Unknown column" in data["error"]
def test_samples_view_rejects_group_by() -> None:
app = server.app
client = app.test_client()
payload = {
"graph_type": "samples",
"group_by": ["user"],
"columns": ["timestamp"],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 400
assert "only valid" in data["error"]
def test_table_avg_with_timestamp() -> None:
app = server.app
client = app.test_client()
payload = {
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "table",
"order_by": "user",
"limit": 100,
"columns": ["user", "timestamp", "value"],
"group_by": ["user"],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert "error" not in data
rows = data["rows"]
assert rows[0][0] == "alice"
from dateutil import parser
ts = parser.parse(rows[0][1]).replace(tzinfo=None)
assert ts == parser.parse("2024-01-01 12:00:00")
def test_timeseries_basic() -> None:
app = server.app
client = app.test_client()
payload = {
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "timeseries",
"limit": 100,
"group_by": ["user"],
"aggregate": "Count",
"columns": ["value"],
"x_axis": "timestamp",
"granularity": "1 day",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert len(data["rows"]) == 4
def test_timeseries_orders_by_xaxis() -> None:
app = server.app
client = app.test_client()
payload = {
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "timeseries",
"limit": 100,
"columns": ["value"],
"x_axis": "timestamp",
"granularity": "1 day",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
rows = data["rows"]
from dateutil import parser
timestamps = [parser.parse(r[0]).replace(tzinfo=None) for r in rows]
assert timestamps == sorted(timestamps)
def test_timeseries_string_column_error() -> None:
app = server.app
client = app.test_client()
payload = {
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "timeseries",
"limit": 7,
"columns": ["timestamp", "event", "value", "user"],
"x_axis": "timestamp",
"granularity": "1 hour",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 400
assert "Aggregate" in data["error"]
def test_derived_column_basic() -> None:
app = server.app
client = app.test_client()
payload = {
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"limit": 10,
"columns": ["timestamp"],
"derived_columns": {"val2": "value * 2"},
"filters": [],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert data["rows"][0][1] == 20
def test_timeseries_derived_column() -> None:
app = server.app
client = app.test_client()
payload = {
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "timeseries",
"granularity": "1 hour",
"limit": 7,
"columns": ["value"],
"derived_columns": {"derived_1": "value * 2"},
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()Hmm, I think the file might be truncated. I'll check the total number of lines.
root@3fe4fa1f8226:/workspace/scubaduck# wc -l tests/test_server.py
500 tests/test_server.pyassert rv.status_code == 200
rows = data["rows"]
assert all(r[2] == r[1] * 2 for r in rows)
def test_default_start_end_returned() -> None:
app = server.app
client = app.test_client()
payload = {
"order_by": "timestamp",
"limit": 5,
"columns": ["timestamp"],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert data["start"] == "2024-01-01 00:00:00"
assert data["end"] == "2024-01-02 03:00:00"page.wait_for_function("window.lastResults !== undefined")
return page.evaluate("window.lastResults")
def test_range_filters(page: Any, server_url: str) -> None:
data = run_query(
page,
server_url,
start="2024-01-02 00:00:00",
end="2024-01-02 04:00:00",
order_by="timestamp",
limit=100,
)
assert len(data["rows"]) == 2
from dateutil import parser
timestamps = [parser.parse(row[0]).replace(tzinfo=None) for row in data["row
s"]]
assert timestamps == [
parser.parse("2024-01-02 00:00:00"),
parser.parse("2024-01-02 03:00:00"),
]
def test_order_by(page: Any, server_url: str) -> None:
data = run_query(
page,
server_url,
start="2024-01-01 00:00:00",
end="2024-01-03 00:00:00",
order_by="value",
order_dir="DESC",
limit=100,
)
values = [row[2] for row in data["rows"]]
assert values == sorted(values, reverse=True)
def test_limit(page: Any, server_url: str) -> None:
data = run_query(
page,
server_url,
start="2024-01-01 00:00:00",
end="2024-01-03 00:00:00",
order_by="timestamp",
limit=2,
)
assert len(data["rows"]) == 2
def test_simple_filter(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
page.click("text=Add Filter")
filter_el = page.query_selector("#filters .filter:last-child")
assert filter_el
page.evaluate(
"arg => setSelectValue(arg.el.querySelector('.f-col'), arg.val)",
{"el": filter_el, "val": "user"},
)
val_input = filter_el.query_selector(".f-val")
val_input.click()
page.keyboard.type("alice")
page.keyboard.press("Enter")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
data = page.evaluate("window.lastResults")
assert len(data["rows"]) == 2
assert all(row[3] == "alice" for row in data["rows"])
def test_default_filter_and_layout(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
count = page.evaluate("document.querySelectorAll('#filters .filter').length"
)
assert count == 1
last_is_button = page.evaluate(
"document.querySelector('#filters').lastElementChild.id === 'add_filter'
"
)
assert last_is_button
position = page.evaluate(
"getComputedStyle(document.querySelector('#filters .filter button.remove
')).position"
)
assert position == "static"
def test_filter_remove_alignment(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
diff = page.evaluate(
"() => { const r=document.querySelector('#filters .filter-row').getBound
ingClientRect(); const x=document.querySelector('#filters .filter-row button.rem
ove').getBoundingClientRect(); return Math.abs(r.right - x.right); }"
)
assert diff <= 1
def test_header_and_tabs(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
header = page.text_content("#header")
assert "sample.csv" in header
assert "events" in header
assert page.is_visible("#settings")
assert page.is_hidden("#columns")
page.click("text=Columns")
assert page.is_visible("#columns")
cols = [c.strip() for c in page.locator("#column_groups li").all_inner_texts
()]
assert "timestamp" in cols
assert "event" in cols
page.click("text=View Settings")
assert page.is_visible("#settings")
btn_color = page.evaluate(
"getComputedStyle(document.querySelector('#dive')).backgroundColor"
)
assert "rgb(0, 128, 0)" == btn_color
...
inp = page.query_selector("#group_by_field .f-val")
assert inp
inp.click()
page.keyboard.type("us")
page.wait_for_selector("#group_by_field .chip-dropdown div")
options = page.locator("#group_by_field .chip-dropdown div").all_inner_texts
()
assert "user" in options
def test_group_by_copy_icon(page: Any, server_url: str) -> None:
page.goto(f"{server_url}?graph_type=table")
page.wait_for_selector("#group_by_field", state="visible")
icon = page.text_content("#group_by_field .chip-copy")
assert icon == "⎘"
def test_group_by_input_no_border(page: Any, server_url: str) -> None:
page.goto(f"{server_url}?graph_type=table")
page.wait_for_selector("#group_by_field", state="visible")
border = page.evaluate(
"getComputedStyle(document.querySelector('#group_by_field .f-val')).bord
erStyle"
)
assert border == "none"
def test_table_group_by_query(page: Any, server_url: str) -> None:
data = run_query(
page,
server_url,
start="2024-01-01 00:00:00",
end="2024-01-03 00:00:00",
order_by="timestamp",
limit=100,
group_by=["user"],
aggregate="Count",
)
assert "error" not in data
assert len(data["rows"]) == 3
def test_table_avg_no_group_by(page: Any, server_url: str) -> None:
data = run_query(
page,
server_url,
aggregate="Avg",
)
assert len(data["rows"]) == 1
row = data["rows"][0]
assert row[0] == 4
from dateutil import parser
ts = parser.parse(row[1]).replace(tzinfo=None)
assert ts == parser.parse("2024-01-01 13:00:00")
assert row[2] == 25
def test_table_headers_show_aggregate(page: Any, server_url: str) -> None:
run_query(
page,
server_url,
aggregate="Avg",
)
headers = page.locator("#results th").all_inner_texts()
assert "Hits" in headers
assert "timestamp (avg)" in headers
assert "value (avg)" in headers
def test_format_number_function(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
vals = page.evaluate(
"() => [formatNumber(815210), formatNumber(999.999), formatNumber(0.0004
), formatNumber(0)]"
)
assert vals == ["815.21 K", "999.999", "0.000", "0"]
def test_derived_column_query(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
page.click("text=Columns")
page.click("text=Add Derived")
expr = page.query_selector("#derived_list .derived textarea")
assert expr
expr.fill("value * 2")
page.click("text=View Settings")
page.fill("#start", "2024-01-01 00:00:00")
page.fill("#end", "2024-01-03 00:00:00")
page.fill("#limit", "10")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
data = page.evaluate("window.lastResults")
assert data["rows"][0][-1] == 20
def test_derived_column_remove(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
page.click("text=Columns")
page.click("text=Add Derived")
assert page.query_selector("#derived_list .derived button.remove")
page.click("#derived_list .derived button.remove")
count = page.evaluate("document.querySelectorAll('#derived_list .derived').l
ength")
assert count == 0
def test_sql_query_display(page: Any, server_url: str) -> None:
data = run_query(
page,
server_url,
start="2024-01-01 00:00:00",
end="2024-01-02 00:00:00",
order_by="timestamp",
limit=10,
)
sql = data["sql"]
displayed = page.text_content("#sql_query")
assert displayed is not None
assert displayed.strip() == sqlI’ll open the entire test_web.py file to examine its structure and orientation.
from typing import Any
def select_value(page: Any, selector: str, value: str) -> None:
page.evaluate(
"arg => setSelectValue(arg.sel, arg.val)",
{"sel": selector, "val": value},
)
def run_query(
page: Any,
url: str,
*,
start: str | None = None,
end: str | None = None,
order_by: str | None = None,
order_dir: str | None = "ASC",
limit: int | None = None,
group_by: list[str] | None = None,
aggregate: str | None = None,
) -> dict[str, Any]:
page.goto(url)
page.wait_for_selector("#order_by option", state="attached")
page.wait_for_selector("#order_dir", state="attached")
page.wait_for_function("window.lastResults !== undefined")
if start is not None:
page.fill("#start", start)
if end is not None:
page.fill("#end", end)
if order_by is not None:
select_value(page, "#order_by", order_by)
if order_dir is not None and order_dir == "DESC":
page.click("#order_dir")
if limit is not None:
page.fill("#limit", str(limit))
if group_by is not None:
select_value(page, "#graph_type", "table"))
def run_query(
page: Any,
url: str,
*,
start: str | None = None,
end: str | None = None,
order_by: str | None = None,
order_dir: str | None = "ASC",
limit: int | None = None,
group_by: list[str] | None = None,
aggregate: str | None = None,
) -> dict[str, Any]:
page.goto(url)
page.wait_for_selector("#order_by option", state="attached")
page.wait_for_selector("#order_dir", state="attached")
page.wait_for_function("window.lastResults !== undefined")
if start is not None:
page.fill("#start", start)
if end is not None:
page.fill("#end", end)
if order_by is not None:
select_value(page, "#order_by", order_by)
if order_dir is not None and order_dir == "DESC":
page.click("#order_dir")
if limit is not None:
page.fill("#limit", str(limit))
if group_by is not None:
select_value(page, "#graph_type", "table")
page.evaluate(
"g => { groupBy.chips = g; groupBy.renderChips(); }",
group_by,
)
if aggregate is not None:
select_value(page, "#graph_type", "table")
select_value(page, "#aggregate", aggregate)
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
return page.evaluate("window.lastResults")
def test_range_filters(page: Any, server_url: str) -> None:
data = run_query(
page,
server_url,
start="2024-01-02 00:00:00",
end="2024-01-02 04:00:00",
order_by="timestamp",
limit=100,
)
assert len(data["rows"]) == 2
from dateutil import parser
timestamps = [parser.parse(row[0]).replace(tzinfo=None) for row in data["row
s"]]
assert timestamps == [
parser.parse("2024-01-02 00:00:00"),
parser.parse("2024-01-02 03:00:00"),
]
def test_order_by(page: Any, server_url: str) -> None:
data = run_query(
page,
server_url,
start="2024-01-01 00:00:00",
end="2024-01-03 00:00:00",
order_by="value",
order_dir="DESC",
limit=100,
)
values = [row[2] for row in data["rows"]]
assert values == sorted(values, reverse=True)
def test_limit(page: Any, server_url: str) -> None:
data = run_query(
page,
server_url,
start="2024-01-01 00:00:00",
end="2024-01-03 00:00:00",
order_by="timestamp",
limit=2,
)
assert len(data["rows"]) == 2
def test_simple_filter(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
page.click("text=Add Filter")
filter_el = page.query_selector("#filters .filter:last-child")
assert filter_el
page.evaluate(
"arg => setSelectValue(arg.el.querySelector('.f-col'), arg.val)",
{"el": filter_el, "val": "user"},
)
val_input = filter_el.query_selector(".f-val")
val_input.click()
page.keyboard.type("alice")
page.keyboard.press("Enter")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
data = page.evaluate("window.lastResults")
assert len(data["rows"]) == 2
assert all(row[3] == "alice" for row in data["rows"])
def test_default_filter_and_layout(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
count = page.evaluate("document.querySelectorAll('#filters .filter').length"
)
assert count == 1
last_is_button = page.evaluate(
"document.querySelector('#filters').lastElementChild.id === 'add_filter'
"
)
assert last_is_button
position = page.evaluate(
"getComputedStyle(document.querySelector('#filters .filter button.remove
')).position"
)
assert position == "static"
def test_filter_remove_alignment(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
diff = page.evaluate(
"() => { const r=document.querySelector('#filters .filter-row').getBound
ingClientRect(); const x=document.querySelector('#filters .filter-row button.rem
ove').getBoundingClientRect(); return Math.abs(r.right - x.right); }"
)
assert diff <= 1
def test_header_and_tabs(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
header = page.text_content("#header")
assert "sample.csv" in header
assert "events" in header
assert page.is_visible("#settings")
assert page.is_hidden("#columns")
page.click("text=Columns")
assert page.is_visible("#columns")
cols = [c.strip() for c in page.locator("#column_groups li").all_inner_texts
()]
assert "timestamp" in cols
assert "event" in cols
page.click("text=View Settings")
assert page.is_visible("#settings")
btn_color = page.evaluate(
"getComputedStyle(document.querySelector('#dive')).backgroundColor"
)
assert "rgb(0, 128, 0)" == btn_color
sidebar_overflow = page.evaluate(
"getComputedStyle(document.querySelector('#sidebar')).overflowY"
)
view_overflow = page.evaluate(
"getComputedStyle(document.querySelector('#view')).overflowY"
)
assert sidebar_overflow == "auto"
assert view_overflow == "auto"
def test_graph_type_table_fields(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#graph_type", state="attached")
select_value(page, "#graph_type", "table")
assert page.is_visible("#group_by_field")
assert page.is_visible("#aggregate_field")
assert page.is_visible("#show_hits_field")
page.click("text=Columns")
assert not page.is_visible("text=Strings:")
def test_graph_type_timeseries_fields(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#graph_type", state="attached")
select_value(page, "#graph_type", "timeseries")
assert page.is_visible("#group_by_field")
assert page.is_visible("#aggregate_field")
assert page.is_visible("#x_axis_field")
assert page.is_visible("#granularity_field")
assert page.is_visible("#fill_field")
def test_timeseries_default_query(page: Any, server_url: str) -> None:
page.goto(server_url)assert not page.is_checked("#column_groups input[value='timestamp']")
def test_timeseries_single_bucket(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#graph_type", state="attached")
page.fill("#start", "2024-01-01 00:00:00")
page.fill("#end", "2024-01-01 00:00:00")
select_value(page, "#graph_type", "timeseries")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
path = page.get_attribute("#chart path", "d")
assert path is not None and "NaN" not in path
def test_timeseries_fill_options(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#graph_type", state="attached")
page.fill("#start", "2024-01-01 00:00:00")
page.fill("#end", "2024-01-02 03:00:00")
select_value(page, "#graph_type", "timeseries")
select_value(page, "#granularity", "1 hour")
select_value(page, "#fill", "0")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
path_zero = page.get_attribute("#chart path", "d")
assert path_zero is not None and path_zero.count("L") > 20
select_value(page, "#fill", "connect")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
path_conn = page.get_attribute("#chart path", "d")
assert path_conn is not None and path_conn.count("M") == 1
select_value(page, "#fill", "blank")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
path_blank = page.get_attribute("#chart path", "d")
assert path_blank is not None and path_blank.count("M") > 1
def test_help_and_alignment(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
titles = page.evaluate(
"Array.from(document.querySelectorAll('#settings .help')).map(e => e.tit
le)"
)
assert any("start/end of the time range" in t for t in titles)
text_align = page.evaluate(
"getComputedStyle(document.querySelector('#settings label')).textAlign"
)
assert text_align == "right"
def test_table_sorting(page: Any, server_url: str) -> None:
run_query(
page,
server_url,
start="2024-01-01 00:00:00",
end="2024-01-03 00:00:00",
order_by="timestamp",
order_dir="ASC",
limit=100,
)
# header alignment
align = page.evaluate(
"getComputedStyle(document.querySelector('#results th')).textAlign"
)
assert align == "right"
header = page.locator("#results th").nth(3)
def values() -> list[str]:
return page.locator("#results td:nth-child(4)").all_inner_texts()
orig_rows = values()
assert orig_rows == ["alice", "bob", "alice", "charlie"]
first_sql = page.evaluate("window.lastResults.sql")
header.click()
assert values() == sorted(orig_rows)
assert header.inner_text().endswith("▲")
color = page.evaluate(
"getComputedStyle(document.querySelector('#results th:nth-child(4)')).co
lor"
)
assert "0, 0, 255" in color
assert page.evaluate("window.lastResults.sql") == first_sql
header.click()
assert values() == sorted(orig_rows, reverse=True)
assert header.inner_text().endswith("▼")
header.click()
assert values() == orig_rows
assert header.inner_text() == "user"
color = page.evaluate(
"getComputedStyle(document.querySelector('#results th:nth-child(4)')).co
lor"
)
assert "0, 0, 255" not in color
def test_relative_dropdown(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
btn = page.query_selector('[data-target="start-select"]')
assert btn
btn.click()
page.click("#start-select div:text('-3 hours')")
assert page.input_value("#start") == "-3 hours"
def test_end_dropdown_now(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
page.click('[data-target="end-select"]')
page.click("#end-select div:text('now')")
assert page.input_value("#end") == "now"
def test_invalid_time_error_shown(page: Any, server_url: str) -> None:
data = run_query(
page,
server_url,
start="nonsense",
end="now",
order_by="timestamp",
)
assert "error" in data
msg = page.text_content("#view")
assert "nonsense" in msg
def test_table_avg_group_by(page: Any, server_url: str) -> None:
data = run_query(
page,
server_url,
start="2024-01-01 00:00:00",
end="2024-01-03 00:00:00",
order_by="timestamp",
group_by=["user"],
aggregate="Avg",
)
assert "error" not in data
assert len(data["rows"]) == 3
def test_column_toggle_and_selection(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
page.click("text=Columns")
page.wait_for_selector("#column_groups input", state="attached")
count = page.evaluate(
"document.querySelectorAll('#column_groups input:checked').length"
)
assert count == 4
page.click("#columns_none")
count = page.evaluate(
"document.querySelectorAll('#column_groups input:checked').length"
)
assert count == 0
page.click("#columns_all")
count = page.evaluate(
"document.querySelectorAll('#column_groups input:checked').length"
)
assert count == 4
page.uncheck("#column_groups input[value='value']")
page.click("text=View Settings")
page.fill("#start", "2024-01-01 00:00:00")
page.fill("#end", "2024-01-02 00:00:00")
select_value(page, "#order_by", "timestamp")
page.fill("#limit", "10")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
data = page.evaluate("window.lastResults")
assert len(data["rows"][0]) == 3
headers = page.locator("#results th").all_inner_texts()
assert "value" not in headers
def test_columns_links_alignment(page: Any, server_url: str) -> None:page.click("text=Columns")
page.wait_for_selector("#column_groups a", state="attached")
tag = page.evaluate("document.querySelector('#column_groups .col-group a').t
agName")
assert tag == "A"
def test_column_group_links_float_right(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
page.click("text=Columns")
page.wait_for_selector("#column_groups .col-group .links", state="attached")
float_val = page.evaluate(
"getComputedStyle(document.querySelector('#column_groups .col-group .lin
ks')).float"
)
assert float_val == "right"
def test_chip_dropdown_navigation(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
page.click("text=Add Filter")
f = page.query_selector("#filters .filter:last-child")
assert f
page.evaluate(
"arg => setSelectValue(arg.el.querySelector('.f-col'), arg.val)",
{"el": f, "val": "user"},
)
inp = f.query_selector(".f-val")
inp.click()
page.wait_for_selector("#filters .filter:last-child .chip-dropdown div")
page.keyboard.type("ali")
page.wait_for_selector("text=alice")
page.keyboard.press("ArrowDown")
page.keyboard.press("Enter")
chips = page.evaluate(
"Array.from(document.querySelectorAll('#filters .filter:last-child .chip
')).map(c => c.firstChild.textContent)"
)
assert chips == ["ali"]
page.click("#filters .filter:last-child .chip .x")
page.wait_for_selector(".chip", state="detached")
def test_chip_copy_and_paste(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
page.evaluate(
"Object.defineProperty(navigator, 'clipboard', {value:{ _data: '', write
Text(t){ this._data = t; }, readText(){ return Promise.resolve(this._data); } }}
)"
)
page.click("text=Add Filter")
f = page.query_selector("#filters .filter:last-child")
assert f
page.evaluate(
"arg => setSelectValue(arg.el.querySelector('.f-col'), arg.val)",
{"el": f, "val": "user"},
)
inp = f.query_selector(".f-val")
inp.click()
page.keyboard.type("alice")
page.keyboard.press("Enter")
inp.click()
page.keyboard.type("bob")
page.keyboard.press("Enter")
f.query_selector(".chip-copy").click()
assert page.evaluate("navigator.clipboard._data") == "alice,bob"
page.evaluate(
"var f=document.querySelector('#filters .filter:last-child'); f.chips=[]
; f.querySelectorAll('.chip').forEach(c=>c.remove())"
)
page.wait_for_selector("#filters .chip", state="detached")
inp.click()
page.evaluate(
"var dt=new DataTransfer(); dt.setData('text/plain','alice,bob'); var e=
new ClipboardEvent('paste',{clipboardData:dt}); document.querySelector('#filters
.filter:last-child .f-val').dispatchEvent(e);"
)
chips = page.evaluate(
"Array.from(document.querySelectorAll('#filters .filter:last-child .chip
')).map(c => c.firstChild.textContent)"
)
assert chips[:2] == ["alice", "bob"]
page.evaluate(
"var f=document.querySelector('#filters .filter:last-child'); f.chips=[]
; f.querySelectorAll('.chip').forEach(c=>c.remove())"
)
page.wait_for_selector("#filters .chip", state="detached")
inp.click()
page.evaluate(
"var dt=new DataTransfer(); dt.setData('text/plain','alice,bob'); var e=
new ClipboardEvent('paste',{clipboardData:dt}); Object.defineProperty(e,'shiftKe
y',{value:true}); document.querySelector('#filters .filter:last-child .f-val').d
ispatchEvent(e);"
)
chips = page.evaluate(
"Array.from(document.querySelectorAll('#filters .filter:last-child .chip
')).map(c => c.firstChild.textContent)"
)
assert chips[-1] == "alice,bob"
def test_chip_dropdown_hides_on_outside_click(page: Any, server_url: str) -> Non
e:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
page.click("text=Add Filter")
f = page.query_selector("#filters .filter:last-child")
assert f
page.evaluate(
"arg => setSelectValue(arg.el.querySelector('.f-col'), arg.val)",
{"el": f, "val": "user"},
)
inp = f.query_selector(".f-val")
inp.click()
page.wait_for_selector("#filters .filter:last-child .chip-dropdown div")
page.click("#header")
page.wait_for_selector("#filters .filter:last-child .chip-dropdown", state="
hidden")
def test_chip_input_no_outline(page: Any, server_url: str) -> None:
...
assert chips == ["alice"]
inp.click()
page.keyboard.type("alice")
page.keyboard.press("Enter")
chips = page.evaluate(
"Array.from(document.querySelectorAll('#filters .filter:last-child .chip
')).map(c => c.firstChild.textContent)"
)
assert chips == []
def test_table_enhancements(page: Any, server_url: str) -> None:
run_query(
page,
server_url,
start="2024-01-01 00:00:00",
end="2024-01-03 00:00:00",
order_by="timestamp",
limit=10,
)
border = page.evaluate(
"getComputedStyle(document.querySelector('#results td')).borderStyle"
)
assert border == "solid"
color1 = page.evaluate(
"getComputedStyle(document.querySelector('#results tr:nth-child(2) td'))
.backgroundColor"
)
color2 = page.evaluate(
"getComputedStyle(document.querySelector('#results tr:nth-child(3) td'))
.backgroundColor"
)
assert color1 != color2
page.hover("#results tr:nth-child(2)")
hover_color = page.evaluate(
"getComputedStyle(document.querySelector('#results tr:nth-child(2) td'))
.backgroundColor"
)
assert hover_color != color1
page.click("#results tr:nth-child(2)")
selected_color = page.evaluate(
"getComputedStyle(document.querySelector('#results tr:nth-child(2) td'))
.backgroundColor"
)
assert "189, 228, 255" in selected_color
overflow = page.evaluate(
"var v=document.getElementById('view'); v.scrollWidth > v.clientWidth"
)
assert not overflow
def test_timestamp_rendering(page: Any, server_url: str) -> None:
run_query(
page,
server_url,
start="2024-01-01 00:00:00",
end="2024-01-02 00:00:00",
order_by="timestamp",
limit=1,
)
cell = page.text_content("#results td")
assert cell != "Invalid Date"
valid = page.evaluate("v => !isNaN(Date.parse(v))", cell)
assert valid
def test_url_query_persistence(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
page.wait_for_function("window.lastResults !== undefined")
page.fill("#start", "2024-01-01 00:00:00")
page.fill("#end", "2024-01-02 00:00:00")
page.fill("#limit", "1")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
first_url = page.url
first_rows = page.evaluate("window.lastResults.rows.length")
page.fill("#limit", "2")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
second_url = page.url
second_rows = page.evaluate("window.lastResults.rows.length")
assert second_rows != first_rows
assert first_url != second_url
page.go_back()
page.wait_for_function("window.lastResults !== undefined")
assert page.url == first_url
assert page.evaluate("window.lastResults.rows.length") == first_rows
def test_load_from_url(page: Any, server_url: str) -> None:
url = (
f"{server_url}?start=2024-01-01%2000:00:00&end=2024-01-02%2000:00:00"
"&order_by=timestamp&limit=2"
)
page.goto(url)
page.wait_for_selector("#order_by option", state="attached")
page.wait_for_function("window.lastResults !== undefined")
assert page.input_value("#start") == "2024-01-01 00:00:00"
assert page.input_value("#end") == "2024-01-02 00:00:00"
assert page.input_value("#limit") == "2"
assert page.evaluate("window.lastResults.rows.length") == 2
def test_empty_data_message(page: Any, server_url: str) -> None:
data = run_query(
page,
server_url,
start="2025-01-01 00:00:00",
end="2025-01-02 00:00:00",
order_by="timestamp",
limit=100,
)
assert data["rows"] == []
msg = page.text_content("#view")
assert "Empty data provided to table" in msgI'm checking through additional lines past 800 to ensure I capture everything.
)
assert chips == ["user"]
def test_group_by_autocomplete(page: Any, server_url: str) -> None:
page.goto(f"{server_url}?graph_type=table")
page.wait_for_selector("#group_by_field", state="visible")
inp = page.query_selector("#group_by_field .f-val")
assert inp
inp.click()
page.keyboard.type("us")
page.wait_for_selector("#group_by_field .chip-dropdown div")
options = page.locator("#group_by_field .chip-dropdown div").all_inner_texts
()
assert "user" in options
def test_group_by_copy_icon(page: Any, server_url: str) -> None:
page.goto(f"{server_url}?graph_type=table")
page.wait_for_selector("#group_by_field", state="visible")
icon = page.text_content("#group_by_field .chip-copy")
assert icon == "⎘"
def test_group_by_input_no_border(page: Any, server_url: str) -> None:
page.goto(f"{server_url}?graph_type=table")
page.wait_for_selector("#group_by_field", state="visible")
border = page.evaluate(
"getComputedStyle(document.querySelector('#group_by_field .f-val')).bord
erStyle"
)
assert border == "none"
def test_table_group_by_query(page: Any, server_url: str) -> None:
data = run_query(
page,
server_url,
start="2024-01-01 00:00:00",
end="2024-01-03 00:00:00",
order_by="timestamp",
limit=100,
group_by=["user"],
aggregate="Count",
)
assert "error" not in data
assert len(data["rows"]) == 3
def test_table_avg_no_group_by(page: Any, server_url: str) -> None:
data = run_query(
page,
server_url,
aggregate="Avg",
)
assert len(data["rows"]) == 1
row = data["rows"][0]
assert row[0] == 4
from dateutil import parser
ts = parser.parse(row[1]).replace(tzinfo=None)
assert ts == parser.parse("2024-01-01 13:00:00")
assert row[2] == 25
def test_table_headers_show_aggregate(page: Any, server_url: str) -> None:
run_query(
page,
server_url,
aggregate="Avg",
)
headers = page.locator("#results th").all_inner_texts()
assert "Hits" in headers
assert "timestamp (avg)" in headers
assert "value (avg)" in headers
def test_format_number_function(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
vals = page.evaluate(
"() => [formatNumber(815210), formatNumber(999.999), formatNumber(0.0004
), formatNumber(0)]"
)
assert vals == ["815.21 K", "999.999", "0.000", "0"]
def test_derived_column_query(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
page.click("text=Columns")
page.click("text=Add Derived")
expr = page.query_selector("#derived_list .derived textarea")
assert expr
expr.fill("value * 2")
page.click("text=View Settings")
page.fill("#start", "2024-01-01 00:00:00")
page.fill("#end", "2024-01-03 00:00:00")
page.fill("#limit", "10")
page.evaluate("window.lastResults = undefined")
page.click("text=Dive")
page.wait_for_function("window.lastResults !== undefined")
data = page.evaluate("window.lastResults")
assert data["rows"][0][-1] == 20
def test_derived_column_remove(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
page.click("text=Columns")
page.click("text=Add Derived")
assert page.query_selector("#derived_list .derived button.remove")
page.click("#derived_list .derived button.remove")
count = page.evaluate("document.querySelectorAll('#derived_list .derived').l
ength")
assert count == 0
def test_sql_query_display(page: Any, server_url: str) -> None:
data = run_query(
page,
server_url,
start="2024-01-01 00:00:00",
end="2024-01-02 00:00:00",
order_by="timestamp",
limit=10,
)
sql = data["sql"]
displayed = page.text_content("#sql_query")
assert displayed is not None
assert displayed.strip() == sqlroot@3fe4fa1f8226:/workspace/scubaduck# wc -l tests/test_web.py
935 tests/test_web.pyThe file has 935 lines. I'll update tests based on what's present in static/index.html.
n 40
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<title>ScubaDuck</title>
<style>
body { margin: 0; font-family: sans-serif; height: 100vh; display: flex; fle
x-direction: column; }
#header { padding: 10px; font-weight: bold; border-bottom: 1px solid #ccc; }
#content { flex: 1; display: flex; height: calc(100vh - 42px); overflow: hid
den; }
#sidebar { width: 450px; padding: 10px; border-right: 3px solid #ccc; overfl
ow-y: auto; display: flex; flex-direction: column; box-sizing: border-box; }
#sidebar-resizer { width: 5px; cursor: col-resize; background: #ccc; }
#view { flex: 1; padding: 10px; overflow-y: auto; overflow-x: auto; }
.field { display: flex; align-items: center; margin-bottom: 10px; }
.field label { width: 80px; text-align: right; margin-right: 5px; }
.help { margin-left: 4px; cursor: help; }
.rel-btn { margin-left: 4px; }
#tabs { display: flex; align-items: center; margin-bottom: 10px; }
#tabs .tab { margin-right: 5px; background: none; border: 1px solid #ccc; pa
dding: 4px 8px; cursor: pointer; width: 120px; text-align: center; box-sizing: b
order-box; }
#tabs .tab.active { background: #eee; font-weight: bold; }
#dive { margin-left: auto; background: green; color: white; border: none; pa
dding: 5px 10px; cursor: pointer; }
.tab-content { display: none; }
.tab-content.active { display: block; }
#filter_list { display: flex; flex-direction: column; }
#filters .filter {
border: 1px solid #ccc;
padding: 5px;
margin-bottom: 5px;
position: relative;
display: flex;
flex-direction: column;
}
#derived_columns .derived {
border: 1px solid #ccc;
padding: 5px;
margin-bottom: 5px;
display: flex;
flex-direction: column;
}
#derived_columns .derived-row {
display: flex;I'm looking for the "sample.csv - events" header near the top.
root@3fe4fa1f8226:/workspace/scubaduck# grep -n "header" -n scubaduck/static/ind
ex.html
8: #header { padding: 10px; font-weight: bold; border-bottom: 1px solid #ccc;
}
128: .col-group-header {
131: .col-group-header .links {
134: .col-group-header .links a {
141: <div id="header">sample.csv - events <select id="graph_type"><option value
="samples">Samples</option><option value="table">Table</option><option value="ti
meseries">Time Series</option></select></div>
437: if (g.querySelector('.col-group-header').textContent.startsWith('Strings
')) {
508: const header = document.createElement('div');
509: header.className = 'col-group-header';
510: header.appendChild(document.createTextNode(g.name + ': '));
521: header.appendChild(links);
522: div.appendChild(header);
828: fetch('/api/query', {method:'POST', headers:{'Content-Type':'application/j
son'}, body:JSON.stringify(payload)})
1015: const header = document.createElement('tr');
1036: header.appendChild(th);
1038: table.appendChild(header);I'll review the header near line 141 to check the graph type and other details.
root@3fe4fa1f8226:/workspace/scubaduck# sed -n '135,152p' scubaduck/static/index
.html
margin-left: 5px;
}
/* Column resizer removed */
</style>
</head>
<body>
<div id="header">sample.csv - events <select id="graph_type"><option value="sa
mples">Samples</option><option value="table">Table</option><option value="timese
ries">Time Series</option></select></div>
<div id="content">
<div id="sidebar">
<div id="tabs">
<button class="tab active" data-tab="settings">View Settings</button>
<button class="tab" data-tab="columns">Columns</button>
<button id="dive" onclick="dive()">Dive</button>
</div>
<div id="settings" class="tab-content active">
<div class="field">
<label>Start<span class="help" title="Sets the start/end of the time r
ange to query. Can be any kind of datetime string. For example: 'April 23, 2014'
or 'yesterday'.">[?]</span></label>
<div class="rel-box">I'll adjust the header to show the table dropdown and find JS functions for querying.
root@3fe4fa1f8226:/workspace/scubaduck# grep -n "dive()" -n scubaduck/static/ind
ex.html
147: <button id="dive" onclick="dive()">Dive</button>root@3fe4fa1f8226:/workspace/scubaduck# grep -n "function dive" -n scubaduck/sta
tic/index.html
813:function dive(push=true) {root@3fe4fa1f8226:/workspace/scubaduck# sed -n '811,858p' scubaduck/static/index
.html
let queryStart = 0;
function dive(push=true) {
const params = collectParams();
if (push) {
history.pushState(params, '', paramsToSearch(params));
}
const payload = Object.assign({}, params);
const dcMap = {};
(params.derived_columns || []).forEach(d => {
if (d.include) dcMap[d.name] = d.expr;
});
payload.derived_columns = dcMap;
const view = document.getElementById('view');
view.innerHTML = '<p>Loading...</p>';
window.lastResults = undefined;
queryStart = performance.now();
fetch('/api/query', {method:'POST', headers:{'Content-Type':'application/json'
}, body:JSON.stringify(payload)})
.then(async r => {
const data = await r.json();
if (!r.ok) throw data;
return data;
})
.then(data => {
lastQueryTime = Math.round(performance.now() - queryStart);
showResults(data);
})
.catch(err => {
showError(err);
});
}
function collectParams() {
updateSelectedColumns();
const payload = {
start: document.getElementById('start').value,
end: document.getElementById('end').value,
order_by: document.getElementById('order_by').value,
order_dir: orderDir,
limit: parseInt(document.getElementById('limit').value, 10),
columns: selectedColumns.filter(c =>
c !== 'Hits' && !derivedColumns.some(dc => dc.name === c)
),
graph_type: graphTypeSel.value,
filters: Array.from(document.querySelectorAll('#filters .filter')).map(f =>
{
const chips = f.chips || [];
const op = f.querySelector('.f-op').value;
let value = null;root@3fe4fa1f8226:/workspace/scubaduck# sed -n '858,930p' scubaduck/static/index
.html
let value = null;
if (op !== 'empty' && op !== '!empty') {
value = chips.length === 0 ? null : (chips.length === 1 ? chips[0] : chi
ps);
}
const colSel = f.querySelector('.f-col select') || f.querySelector('.f-col
');
return {column: colSel.value, op, value};
}),
derived_columns: Array.from(document.querySelectorAll('#derived_list .derive
d')).map(d => ({
type: d.querySelector('.d-type').value,
name: d.querySelector('.d-name').value,
expr: d.querySelector('.d-expr').value,
include: d.querySelector('.d-use').checked,
}))
};
if (graphTypeSel.value === 'table' || graphTypeSel.value === 'timeseries') {
payload.group_by = groupBy.chips || [];
payload.aggregate = document.getElementById('aggregate').value;
payload.show_hits = document.getElementById('show_hits').checked;
}
if (graphTypeSel.value === 'timeseries') {
payload.x_axis = document.getElementById('x_axis').value;
payload.granularity = document.getElementById('granularity').value;
payload.fill = document.getElementById('fill').value;
}
return payload;
}
function paramsToSearch(params) {
const sp = new URLSearchParams();
if (params.start) sp.set('start', params.start);
if (params.end) sp.set('end', params.end);
if (params.order_by) sp.set('order_by', params.order_by);
if (params.order_dir) sp.set('order_dir', params.order_dir);
if (params.limit !== null && params.limit !== undefined) sp.set('limit', param
s.limit);
if (params.columns && params.columns.length) sp.set('columns', params.columns.
join(','));
if (params.filters && params.filters.length) sp.set('filters', JSON.stringify(
params.filters));
if (params.derived_columns && params.derived_columns.length) sp.set('derived_c
olumns', JSON.stringify(params.derived_columns));
if (params.graph_type) sp.set('graph_type', params.graph_type);
if (params.graph_type === 'table' || params.graph_type === 'timeseries') {
if (params.group_by && params.group_by.length) sp.set('group_by', params.gro
up_by.join(','));
if (params.aggregate) sp.set('aggregate', params.aggregate);
if (params.show_hits) sp.set('show_hits', '1');
}
if (params.graph_type === 'timeseries') {
if (params.x_axis) sp.set('x_axis', params.x_axis);
if (params.granularity) sp.set('granularity', params.granularity);
if (params.fill) sp.set('fill', params.fill);
}
const qs = sp.toString();
return qs ? '?' + qs : '';
}
function applyParams(params) {
document.getElementById('start').value = params.start || '';
document.getElementById('end').value = params.end || '';
if (params.order_by) {
document.getElementById('order_by').value = params.order_by;
}
orderDir = params.order_dir || 'ASC';
updateOrderDirButton();
if (params.limit !== undefined && params.limit !== null) {
document.getElementById('limit').value = params.limit;
}
graphTypeSel.value = params.graph_type || 'samples';
updateDisplayTypeUI();
if (params.x_axis) document.getElementById('x_axis').value = params.x_axis;
if (params.granularity) document.getElementById('granularity').value = params.
granularity;
if (params.fill) document.getElementById('fill').value = params.fill;
if (params.group_by) {
groupBy.chips.splice(0, groupBy.chips.length, ...params.group_by);
groupBy.renderChips();
}
if (params.aggregate) document.getElementById('aggregate').value = params.aggr
egate;I'm checking from line 930 to view the remaining functions.
root@3fe4fa1f8226:/workspace/scubaduck# sed -n '930,1040p' scubaduck/static/inde
x.html
if (params.aggregate) document.getElementById('aggregate').value = params.aggr
egate;
document.getElementById('show_hits').checked = params.show_hits ?? true;
document.querySelectorAll('#column_groups input').forEach(cb => {
cb.checked = !params.columns || params.columns.includes(cb.value);
});
updateSelectedColumns();
const dlist = document.getElementById('derived_list');
dlist.innerHTML = '';
derivedColumns.splice(0, derivedColumns.length);
if (params.derived_columns && params.derived_columns.length) {
params.derived_columns.forEach(dc => addDerived(dc));
}
refreshDerivedColumns();
const list = document.getElementById('filter_list');
list.innerHTML = '';
if (params.filters && params.filters.length) {
params.filters.forEach(f => {
addFilter();
const el = list.lastElementChild;
const colSel = el.querySelector('.f-col select') || el.querySelector('.f-c
ol');
colSel.value = f.column;
colSel.dispatchEvent(new Event('change'));
el.querySelector('.f-op').value = f.op;
el.querySelector('.f-op').dispatchEvent(new Event('change'));
if (f.value !== null && f.op !== 'empty' && f.op !== '!empty') {
const values = Array.isArray(f.value) ? f.value : [f.value];
values.forEach(v => el.addChip(v));
el.renderChips();
}
});
} else {
addFilter();
}
}
function parseSearch() {
const sp = new URLSearchParams(window.location.search);
const params = {};
if (sp.has('start')) params.start = sp.get('start');
if (sp.has('end')) params.end = sp.get('end');
if (sp.has('order_by')) params.order_by = sp.get('order_by');
if (sp.has('order_dir')) params.order_dir = sp.get('order_dir');
if (sp.has('limit')) params.limit = parseInt(sp.get('limit'), 10);
if (sp.has('columns')) params.columns = sp.get('columns').split(',').filter(c
=> c);
if (sp.has('filters')) {
try { params.filters = JSON.parse(sp.get('filters')); } catch(e) { params.fi
lters = []; }
}
if (sp.has('graph_type')) params.graph_type = sp.get('graph_type');
if (sp.has('group_by')) params.group_by = sp.get('group_by').split(',').filter
(c => c);
if (sp.has('aggregate')) params.aggregate = sp.get('aggregate');
if (sp.has('show_hits')) params.show_hits = sp.get('show_hits') === '1';
if (sp.has('x_axis')) params.x_axis = sp.get('x_axis');
if (sp.has('granularity')) params.granularity = sp.get('granularity');
if (sp.has('fill')) params.fill = sp.get('fill');
if (sp.has('derived_columns')) {
try { params.derived_columns = JSON.parse(sp.get('derived_columns')); } catc
h(e) { params.derived_columns = []; }
}
return params;
}
function initFromUrl() {
const params = parseSearch();
history.replaceState(params, '', paramsToSearch(params));
applyParams(params);
dive(false);
}
window.addEventListener('popstate', e => {
const params = e.state || parseSearch();
applyParams(params);
dive(false);
});
let originalRows = [];
let sortState = {index: null, dir: null};
function renderTable(rows) {
const table = document.getElementById('results');
table.innerHTML = '';
if (rows.length === 0) return;
let hitsIndex = selectedColumns.indexOf('Hits');
let totalHits = 0;
if (hitsIndex !== -1) {
totalHits = rows.reduce((s, r) => s + Number(r[hitsIndex]), 0);
}
const header = document.createElement('tr');
selectedColumns.forEach((col, i) => {
const th = document.createElement('th');
let label = col;
if (
displayType === 'table' &&
col !== 'Hits' &&
!(groupBy.chips || []).includes(col)
) {
const agg = document.getElementById('aggregate').value.toLowerCase();
label += ` (${agg})`;
}
th.textContent = label;
th.dataset.index = i;
th.addEventListener('click', handleSort);
if (sortState.index === i) {
th.classList.add('sorted');
th.textContent =
label + (sortState.dir === 'desc' ? ' \u25BC' : ' \u25B2');
}
if (!isStringColumn(col)) th.style.textAlign = 'right';
header.appendChild(th);
});
table.appendChild(header);
rows.forEach(row => {
const tr = document.createElement('tr');I'll need to modify the fetch call to include the 'table' parameter and find column data.
root@3fe4fa1f8226:/workspace/scubaduck# grep -n "/api/columns" -n -C2 scubaduck/
static/index.html
460-updateOrderDirButton();
461-graphTypeSel.addEventListener('change', updateDisplayTypeUI);
462:fetch('/api/columns').then(r => r.json()).then(cols => {
463- const orderSelect = document.getElementById('order_by');
464- const xAxisSelect = document.getElementById('x_axis');root@3fe4fa1f8226:/workspace/scubaduck# sed -n '462,550p' scubaduck/static/index
.html
fetch('/api/columns').then(r => r.json()).then(cols => {
const orderSelect = document.getElementById('order_by');
const xAxisSelect = document.getElementById('x_axis');
const groupsEl = document.getElementById('column_groups');
const groups = {
time: {name: 'Time', cols: [], ul: null},
integer: {name: 'Integers', cols: [], ul: null},
string: {name: 'Strings', cols: [], ul: null}
};
cols.forEach(c => {
const t = c.type.toUpperCase();
columnTypes[c.name] = c.type;
allColumns.push(c.name);
baseColumns.push(c.name);
let g = 'string';
if (t.includes('INT')) g = 'integer';
if (t.includes('TIMESTAMP')) {
g = 'time';
timeColumns.push(c.name);
baseTimeColumns.push(c.name);
}
if (g === 'string') {
stringColumns.push(c.name);
baseStringColumns.push(c.name);
} else {
integerColumns.push(c.name);
baseIntegerColumns.push(c.name);
}
groups[g].cols.push(c.name);
if (g !== 'string') {
const o = document.createElement('option');
o.value = c.name;
o.textContent = c.name;
orderSelect.appendChild(o);
}
});
timeColumns.forEach(name => {
const o = document.createElement('option');
o.value = name;
o.textContent = name;
xAxisSelect.appendChild(o);
});
Object.keys(groups).forEach(key => {
const g = groups[key];
const div = document.createElement('div');
div.className = 'col-group';
const header = document.createElement('div');
header.className = 'col-group-header';
header.appendChild(document.createTextNode(g.name + ': '));
const links = document.createElement('span');
links.className = 'links';
const allBtn = document.createElement('a');
allBtn.href = '#';
allBtn.textContent = 'All';
const noneBtn = document.createElement('a');
noneBtn.href = '#';
noneBtn.textContent = 'None';
links.appendChild(allBtn);
links.appendChild(noneBtn);
header.appendChild(links);
div.appendChild(header);
const ul = document.createElement('ul');
g.ul = ul;
g.cols.forEach(name => {
const li = document.createElement('li');
const label = document.createElement('label');
const cb = document.createElement('input');
cb.type = 'checkbox';
cb.value = name;
cb.checked = true;
cb.addEventListener('change', updateSelectedColumns);
label.appendChild(cb);
label.appendChild(document.createTextNode(' ' + name));
li.appendChild(label);
ul.appendChild(li);
});
allBtn.addEventListener('click', e => {
e.preventDefault();
ul.querySelectorAll('input').forEach(cb => (cb.checked = true));
updateSelectedColumns();
});
noneBtn.addEventListener('click', e => {
e.preventDefault();
ul.querySelectorAll('input').forEach(cb => (cb.checked = false));
updateSelectedColumns();
});
div.appendChild(ul);
groupsEl.appendChild(div);
});root@3fe4fa1f8226:/workspace/scubaduck# sed -n '550,660p' scubaduck/static/index
.html
});
document.getElementById('columns_all').addEventListener('click', e => {
e.preventDefault();
groupsEl.querySelectorAll('input').forEach(cb => (cb.checked = true));
updateSelectedColumns();
});
document.getElementById('columns_none').addEventListener('click', e => {
e.preventDefault();
groupsEl.querySelectorAll('input').forEach(cb => (cb.checked = false));
updateSelectedColumns();
});
updateSelectedColumns();
groupBy = document.getElementById('group_by').closest('.field');
initChipInput(groupBy, typed =>
allColumns.filter(c => c.toLowerCase().includes(typed.toLowerCase()))
);
initDropdown(orderSelect);
initDropdown(document.getElementById('aggregate'));
document.getElementById('limit').addEventListener('input', e => {
e.target.dataset.setByUser = '1';
});
updateDisplayTypeUI();
addFilter();
initFromUrl();
});
document.querySelectorAll('#tabs .tab').forEach(btn => {
btn.addEventListener('click', () => {
document.querySelectorAll('#tabs .tab').forEach(t => t.classList.remove('act
ive'));
document.querySelectorAll('.tab-content').forEach(c => c.classList.remove('a
ctive'));
btn.classList.add('active');
document.getElementById(btn.dataset.tab).classList.add('active');
});
});
document.querySelectorAll('.rel-btn').forEach(btn => {
btn.addEventListener('click', () => {
const dd = document.getElementById(btn.dataset.target);
const show = dd.style.display === 'none' || dd.style.display === '';
document.querySelectorAll('.rel-dropdown').forEach(d => (d.style.display = '
none'));
dd.style.display = show ? 'block' : 'none';
});
});
document.querySelectorAll('.rel-dropdown div').forEach(opt => {
opt.addEventListener('click', () => {
const box = opt.closest('.rel-box');
const input = box.querySelector('input');
input.value = opt.dataset.value || opt.textContent;
opt.parentElement.style.display = 'none';
});
});
document.addEventListener('click', e => {
document.querySelectorAll('.rel-dropdown').forEach(dd => {
if (!dd.parentElement.contains(e.target)) dd.style.display = 'none';
});
});
function updateSelectedColumns() {
const base = allColumns.filter(name => {
const cb = document.querySelector(`#column_groups input[value="${name}"]`);
if (!cb || !cb.checked) return false;
if (graphTypeSel.value === 'table' && isStringColumn(name)) return false;
return true;
});
if (graphTypeSel.value === 'table' || graphTypeSel.value === 'timeseries') {
selectedColumns = groupBy.chips.slice();
if (document.getElementById('show_hits').checked) selectedColumns.push('Hits
');
base.forEach(c => {
if (!selectedColumns.includes(c)) selectedColumns.push(c);
});
derivedColumns.forEach(dc => {
if (dc.include && !selectedColumns.includes(dc.name)) selectedColumns.push
(dc.name);
});
} else {
selectedColumns = base.slice();
derivedColumns.forEach(dc => {
if (dc.include) selectedColumns.push(dc.name);
});
}
}
function isStringColumn(name) {
const t = (columnTypes[name] || '').toUpperCase();
return t.includes('CHAR') || t.includes('STRING') || t.includes('VARCHAR');
}
function isIntegerColumn(name) {
const t = (columnTypes[name] || '').toUpperCase();
return t.includes('INT');
}
function isTimeColumn(name) {
const t = (columnTypes[name] || '').toUpperCase();
return t.includes('TIMESTAMP');
}
function formatNumber(val) {
if (typeof val !== 'number') val = Number(val);
if (Number.isNaN(val)) return '';
if (val === 0) return '0';
const abs = Math.abs(val);
if (abs > 999.999) {
const units = [
{n: 1e12, s: 'T'},
{n: 1e9, s: 'B'},
{n: 1e6, s: 'M'},
{n: 1e3, s: 'K'},
];
for (const u of units) {
if (abs >= u.n) {
return (val / u.n).toFixed(2) + ' ' + u.s;I'll include the 'table' parameter in the initialization and handle it in parseSearch.
root@3fe4fa1f8226:/workspace/scubaduck# grep -n "sample.csv" -n scubaduck/static
/index.html
141: <div id="header">sample.csv - events <select id="graph_type"><option value
="samples">Samples</option><option value="table">Table</option><option value="ti
meseries">Time Series</option></select></div>I’ll look for any fetch calls or variables related to headers, starting from the bottom.
root@3fe4fa1f8226:/workspace/scubaduck# grep -n "view")" -n scubaduck/static/ind
ex.html
bash: syntax error near unexpected token `)'I'll look for "dbName," "table," or "columns" in the code.
root@3fe4fa1f8226:/workspace/scubaduck# grep -n "table" -n scubaduck/static/inde
x.html | head
114: table { border-collapse: collapse; min-width: 100%; }
141: <div id="header">sample.csv - events <select id="graph_type"><option value
="samples">Samples</option><option value="table">Table</option><option value="ti
meseries">Time Series</option></select></div>
193: <label>Limit<span class="help" title="Choose the maximum number of
results to show in the chart after any aggregations have been applied. For exam
ple, a limit of 10 will show no more than 10 rows for a table, etc.">[?]</span><
/label>
288: <table id="results"></table>
428: const showTable = graphTypeSel.value === 'table';
611: if (graphTypeSel.value === 'table' && isStringColumn(name)) return false
;
614: if (graphTypeSel.value === 'table' || graphTypeSel.value === 'timeseries')
{
872: if (graphTypeSel.value === 'table' || graphTypeSel.value === 'timeseries')
{
896: if (params.graph_type === 'table' || params.graph_type === 'timeseries') {
1007: const table = document.getElementById('results');root@3fe4fa1f8226:/workspace/scubaduck# grep -n "api/tables" -r scubaduck | headI need to decide if I'll extend '/api/columns' or add a new '/api/tables'. The goal is to add a 'table' query parameter to switch between tables easily, without server restart. I'll need to add a dropdown in the UI and modify existing queries. I'll also adjust for DuckDB's SQLite extension.
I’m exploring attaching SQLite to DuckDB, using views for automatic table mapping without copying.
I'll implement _load_database to handle CSV and SQLite files, creating views for easier access.
root@3fe4fa1f8226:/workspace/scubaduck# grep -n "sample.csv" -n tests/test_web.p
y
149: assert "sample.csv" in headerposition = page.evaluate(
"getComputedStyle(document.querySelector('#filters .filter button.remove
')).position"
)
assert position == "static"
def test_filter_remove_alignment(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
diff = page.evaluate(
"() => { const r=document.querySelector('#filters .filter-row').getBound
ingClientRect(); const x=document.querySelector('#filters .filter-row button.rem
ove').getBoundingClientRect(); return Math.abs(r.right - x.right); }"
)
assert diff <= 1
def test_header_and_tabs(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#order_by option", state="attached")
header = page.text_content("#header")
assert "sample.csv" in header
assert "events" in header
assert page.is_visible("#settings")
assert page.is_hidden("#columns")
page.click("text=Columns")
assert page.is_visible("#columns")
cols = [c.strip() for c in page.locator("#column_groups li").all_inner_texts
()]
assert "timestamp" in cols
assert "event" in cols
page.click("text=View Settings")
assert page.is_visible("#settings")
btn_color = page.evaluate(
"getComputedStyle(document.querySelector('#dive')).backgroundColor"
)
assert "rgb(0, 128, 0)" == btn_colorThe header in index.html is static, but it might need dynamic elements like table selections.
root@3fe4fa1f8226:/workspace/scubaduck# grep -n "sample.csv" -n scubaduck/static
/index.html
141: <div id="header">sample.csv - events <select id="graph_type"><option value
="samples">Samples</option><option value="table">Table</option><option value="ti
meseries">Time Series</option></select></div>The header will stay static with "sample.csv", but I'll add a table and graph dropdowns.
root@3fe4fa1f8226:/workspace/scubaduck# grep -n "columnTypes" -n scubaduck/stati
c/index.html
296:const columnTypes = {};
473: columnTypes[c.name] = c.type;
632: const t = (columnTypes[name] || '').toUpperCase();
637: const t = (columnTypes[name] || '').toUpperCase();
642: const t = (columnTypes[name] || '').toUpperCase();
800: columnTypes[d.name] = d.type;I’ll examine lines 296 to 304 for variable initialization and related logic.
root@3fe4fa1f8226:/workspace/scubaduck# sed -n '288,310p' scubaduck/static/index
.html
<table id="results"></table>
</div>
</div>
<script src="/js/chip_input.js"></script>
<script src="/js/timeseries_chart.js"></script>
<script>
const allColumns = [];
const baseColumns = [];
const columnTypes = {};
const stringColumns = [];
const baseStringColumns = [];
const integerColumns = [];
const baseIntegerColumns = [];
const timeColumns = [];
const baseTimeColumns = [];
const derivedColumns = [];
let selectedColumns = [];
let displayType = 'samples';
let groupBy = {chips: [], addChip: () => {}, renderChips: () => {}};
function initDropdown(select) {
const wrapper = document.createElement('div');
wrapper.className = 'dropdown';root@3fe4fa1f8226:/workspace/scubaduck# sed -n '452,560p' scubaduck/static/index
.html
updateSelectedColumns();
}
displayType = graphTypeSel.value;
}
orderDirBtn.addEventListener('click', () => {
orderDir = orderDir === 'ASC' ? 'DESC' : 'ASC';
updateOrderDirButton();
});
updateOrderDirButton();
graphTypeSel.addEventListener('change', updateDisplayTypeUI);
fetch('/api/columns').then(r => r.json()).then(cols => {
const orderSelect = document.getElementById('order_by');
const xAxisSelect = document.getElementById('x_axis');
const groupsEl = document.getElementById('column_groups');
const groups = {
time: {name: 'Time', cols: [], ul: null},
integer: {name: 'Integers', cols: [], ul: null},
string: {name: 'Strings', cols: [], ul: null}
};
cols.forEach(c => {
const t = c.type.toUpperCase();
columnTypes[c.name] = c.type;
allColumns.push(c.name);
baseColumns.push(c.name);
let g = 'string';
if (t.includes('INT')) g = 'integer';
if (t.includes('TIMESTAMP')) {
g = 'time';
timeColumns.push(c.name);
baseTimeColumns.push(c.name);
}
if (g === 'string') {
stringColumns.push(c.name);
baseStringColumns.push(c.name);
} else {
integerColumns.push(c.name);
baseIntegerColumns.push(c.name);
}
groups[g].cols.push(c.name);
if (g !== 'string') {
const o = document.createElement('option');
o.value = c.name;
o.textContent = c.name;
orderSelect.appendChild(o);
}
});
timeColumns.forEach(name => {
const o = document.createElement('option');
o.value = name;
o.textContent = name;
xAxisSelect.appendChild(o);
});
Object.keys(groups).forEach(key => {
const g = groups[key];
const div = document.createElement('div');
div.className = 'col-group';
const header = document.createElement('div');
header.className = 'col-group-header';
header.appendChild(document.createTextNode(g.name + ': '));
const links = document.createElement('span');
links.className = 'links';
const allBtn = document.createElement('a');
allBtn.href = '#';
allBtn.textContent = 'All';
const noneBtn = document.createElement('a');
noneBtn.href = '#';
noneBtn.textContent = 'None';
links.appendChild(allBtn);
links.appendChild(noneBtn);
header.appendChild(links);
div.appendChild(header);
const ul = document.createElement('ul');
g.ul = ul;
g.cols.forEach(name => {
const li = document.createElement('li');
const label = document.createElement('label');
const cb = document.createElement('input');
cb.type = 'checkbox';
cb.value = name;
cb.checked = true;
cb.addEventListener('change', updateSelectedColumns);
label.appendChild(cb);
label.appendChild(document.createTextNode(' ' + name));
li.appendChild(label);
ul.appendChild(li);
});
allBtn.addEventListener('click', e => {
e.preventDefault();
ul.querySelectorAll('input').forEach(cb => (cb.checked = true));
updateSelectedColumns();
});
noneBtn.addEventListener('click', e => {
e.preventDefault();
ul.querySelectorAll('input').forEach(cb => (cb.checked = false));
updateSelectedColumns();
});
div.appendChild(ul);
groupsEl.appendChild(div);
});
document.getElementById('columns_all').addEventListener('click', e => {
e.preventDefault();
groupsEl.querySelectorAll('input').forEach(cb => (cb.checked = true));
updateSelectedColumns();
});
document.getElementById('columns_none').addEventListener('click', e => {
e.preventDefault();
groupsEl.querySelectorAll('input').forEach(cb => (cb.checked = false));
updateSelectedColumns();
});root@3fe4fa1f8226:/workspace/scubaduck# sed -n '560,640p' scubaduck/static/index
.html
});
updateSelectedColumns();
groupBy = document.getElementById('group_by').closest('.field');
initChipInput(groupBy, typed =>
allColumns.filter(c => c.toLowerCase().includes(typed.toLowerCase()))
);
initDropdown(orderSelect);
initDropdown(document.getElementById('aggregate'));
document.getElementById('limit').addEventListener('input', e => {
e.target.dataset.setByUser = '1';
});
updateDisplayTypeUI();
addFilter();
initFromUrl();
});
document.querySelectorAll('#tabs .tab').forEach(btn => {
btn.addEventListener('click', () => {
document.querySelectorAll('#tabs .tab').forEach(t => t.classList.remove('act
ive'));
document.querySelectorAll('.tab-content').forEach(c => c.classList.remove('a
ctive'));
btn.classList.add('active');
document.getElementById(btn.dataset.tab).classList.add('active');
});
});
document.querySelectorAll('.rel-btn').forEach(btn => {
btn.addEventListener('click', () => {
const dd = document.getElementById(btn.dataset.target);
const show = dd.style.display === 'none' || dd.style.display === '';
document.querySelectorAll('.rel-dropdown').forEach(d => (d.style.display = '
none'));
dd.style.display = show ? 'block' : 'none';
});
});
document.querySelectorAll('.rel-dropdown div').forEach(opt => {
opt.addEventListener('click', () => {
const box = opt.closest('.rel-box');
const input = box.querySelector('input');
input.value = opt.dataset.value || opt.textContent;
opt.parentElement.style.display = 'none';
});
});
document.addEventListener('click', e => {
document.querySelectorAll('.rel-dropdown').forEach(dd => {
if (!dd.parentElement.contains(e.target)) dd.style.display = 'none';
});
});
function updateSelectedColumns() {
const base = allColumns.filter(name => {
const cb = document.querySelector(`#column_groups input[value="${name}"]`);
if (!cb || !cb.checked) return false;
if (graphTypeSel.value === 'table' && isStringColumn(name)) return false;
return true;
});
if (graphTypeSel.value === 'table' || graphTypeSel.value === 'timeseries') {
selectedColumns = groupBy.chips.slice();
if (document.getElementById('show_hits').checked) selectedColumns.push('Hits
');
base.forEach(c => {
if (!selectedColumns.includes(c)) selectedColumns.push(c);
});
derivedColumns.forEach(dc => {
if (dc.include && !selectedColumns.includes(dc.name)) selectedColumns.push
(dc.name);
});
} else {
selectedColumns = base.slice();
derivedColumns.forEach(dc => {
if (dc.include) selectedColumns.push(dc.name);
});
}
}
function isStringColumn(name) {
const t = (columnTypes[name] || '').toUpperCase();
return t.includes('CHAR') || t.includes('STRING') || t.includes('VARCHAR');
}
function isIntegerColumn(name) {
const t = (columnTypes[name] || '').toUpperCase();
return t.includes('INT');
}I'll refactor the code to load columns and fetch tables dynamically for better flexibility.
root@3fe4fa1f8226:/workspace/scubaduck# grep -n "FROM events" -n scubaduck/serve
r.py
63: for row in sconn.execute("SELECT * FROM events"):
209: query = f"SELECT {select_clause} FROM events"
317: f"SELECT DISTINCT {column} FROM events WHERE CAST({column} AS VA
RCHAR) ILIKE '%' || ? || '%' LIMIT 20",
439: con.execute(f"SELECT min({axis}), max({axis}) FROM events").
fetchall()[Let me try opening around 200 lines to examine build_query and any other details.
root@3fe4fa1f8226:/workspace/scubaduck# sed -n '198,215p' scubaduck/server.py
if params.order_by:
query += f" ORDER BY {params.order_by} {params.order_dir}"
elif params.graph_type == "timeseries":
query += " ORDER BY bucket"
if params.limit is not None:
query += f" LIMIT {params.limit}"
return query
for name, expr in params.derived_columns.items():
select_parts.append(f"{expr} AS {name}")
select_clause = ", ".join(select_parts) if select_parts else "*"
query = f"SELECT {select_clause} FROM events"
where_parts: list[str] = []
if params.start:
where_parts.append(f"timestamp >= '{params.start}'")
if params.end:
where_parts.append(f"timestamp <= '{params.end}'")
for f in params.filters:I'll change "FROM events" to a variable and check for other "events" references.
root@3fe4fa1f8226:/workspace/scubaduck# grep -n "events" -n scubaduck/server.py
| head
54: f"CREATE TABLE events AS SELECT * FROM read_csv_auto('{path.as_po
six()}')"
59: info = sconn.execute("PRAGMA table_info(events)").fetchall()
61: con.execute(f"CREATE TABLE events ({col_defs})")
63: for row in sconn.execute("SELECT * FROM events"):
64: con.execute(f"INSERT INTO events VALUES ({placeholders})", row)
209: query = f"SELECT {select_clause} FROM events"
263: r[1]: r[2] for r in con.execute("PRAGMA table_info(events)").fetchal
l()
283: rows = con.execute("PRAGMA table_info(events)").fetchall()
317: f"SELECT DISTINCT {column} FROM events WHERE CAST({column} AS VA
RCHAR) ILIKE '%' || ? || '%' LIMIT 20",
439: con.execute(f"SELECT min({axis}), max({axis}) FROM events").
fetchall()[I'll open lines 209 to 245 to confirm the changes around the 200 mark.
root@3fe4fa1f8226:/workspace/scubaduck# sed -n '215,265p' scubaduck/server.py
for f in params.filters:
op = f.op
if op in {"empty", "!empty"}:
val = "''"
else:
if f.value is None:
continue
if isinstance(f.value, list):
if not f.value:
continue
if op == "=":
vals = " OR ".join(f"{f.column} = '{v}'" for v in f.value)
where_parts.append(f"({vals})")
continue
val = f"'{f.value}'" if isinstance(f.value, str) else str(f.value)
if op == "contains":
where_parts.append(f"{f.column} ILIKE '%' || {val} || '%'")
elif op == "!contains":
where_parts.append(f"{f.column} NOT ILIKE '%' || {val} || '%'")
elif op == "empty":
where_parts.append(f"{f.column} = {val}")
elif op == "!empty":
where_parts.append(f"{f.column} != {val}")
else:
where_parts.append(f"{f.column} {op} {val}")
if where_parts:
query += " WHERE " + " AND ".join(where_parts)
if group_cols:
query += " GROUP BY " + ", ".join(group_cols)
if params.order_by:
query += f" ORDER BY {params.order_by} {params.order_dir}"
elif params.graph_type == "timeseries":
query += " ORDER BY bucket"
if params.limit is not None:
query += f" LIMIT {params.limit}"
return query
def create_app(db_file: str | Path | None = None) -> Flask:
app = Flask(__name__, static_folder="static")
if db_file is None:
env_db = os.environ.get("SCUBADUCK_DB")
if env_db:
db_file = env_db
db_path = Path(db_file or Path(__file__).with_name("sample.csv")).resolve()
con = _load_database(db_path)
column_types: Dict[str, str] = {
r[1]: r[2] for r in con.execute("PRAGMA table_info(events)").fetchall()
}sample_cache: Dict[Tuple[str, str], Tuple[List[str], float]] = {}
CACHE_TTL = 60.0
CACHE_LIMIT = 200
@app.route("/")
def index() -> Any: # pyright: ignore[reportUnusedFunction]
assert app.static_folder is not None
return send_from_directory(app.static_folder, "index.html")
@app.route("/js/<path:filename>")
def js(filename: str) -> Any: # pyright: ignore[reportUnusedFunction]
assert app.static_folder is not None
folder = Path(app.static_folder) / "js"
return send_from_directory(folder, filename)
@app.route("/api/columns")
def columns() -> Any: # pyright: ignore[reportUnusedFunction]
rows = con.execute("PRAGMA table_info(events)").fetchall()
return jsonify([{"name": r[1], "type": r[2]} for r in rows])
def _cache_get(key: Tuple[str, str]) -> List[str] | None:
item = sample_cache.get(key)
if item is None:
return None
vals, ts = item
if time.time() - ts > CACHE_TTL:
del sample_cache[key]
return None
sample_cache[key] = (vals, time.time())
return vals
def _cache_set(key: Tuple[str, str], vals: List[str]) -> None:
sample_cache[key] = (vals, time.time())
if len(sample_cache) > CACHE_LIMIT:
oldest = min(sample_cache.items(), key=lambda kv: kv[1][1])[0]
del sample_cache[oldest]
@app.route("/api/samples")
def sample_values() -> Any: # pyright: ignore[reportUnusedFunction]
column = request.args.get("column")
substr = request.args.get("q", "")
if not column or column not in column_types:
return jsonify([])
ctype = column_types[column].upper()
if "CHAR" not in ctype and "STRING" not in ctype and "VARCHAR" not in ct
ype:
return jsonify([])
key = (column, substr)
cached = _cache_get(key)
if cached is not None:
return jsonify(cached)
rows = con.execute(
f"SELECT DISTINCT {column} FROM events WHERE CAST({column} AS VARCHA
R) ILIKE '%' || ? || '%' LIMIT 20",
[substr],
).fetchall()
values = [r[0] for r in rows]
_cache_set(key, values)
return jsonify(values)
@app.route("/api/query", methods=["POST"])
def query() -> Any: # pyright: ignore[reportUnusedFunction]
payload = request.get_json(force=True)
try:
start = parse_time(payload.get("start"))
end = parse_time(payload.get("end"))
except Exception as exc:
return jsonify({"error": str(exc)}), 400
params = QueryParams(
start=start,
end=end,
order_by=payload.get("order_by"),
order_dir=payload.get("order_dir", "ASC"),
limit=payload.get("limit"),
columns=payload.get("columns", []),
derived_columns=payload.get("derived_columns", {}),root@3fe4fa1f8226:/workspace/scubaduck# sed -n '340,460p' scubaduck/server.py
derived_columns=payload.get("derived_columns", {}),
graph_type=payload.get("graph_type", "samples"),
group_by=payload.get("group_by", []),
aggregate=payload.get("aggregate"),
show_hits=payload.get("show_hits", False),
x_axis=payload.get("x_axis"),
granularity=payload.get("granularity", "Auto"),
fill=payload.get("fill", "0"),
)
for f in payload.get("filters", []):
params.filters.append(Filter(f["column"], f["op"], f.get("value")))
if params.graph_type not in {"table", "timeseries"} and (
params.group_by or params.aggregate or params.show_hits
):
return (
jsonify(
{
"error": "group_by, aggregate and show_hits are only val
id for table or timeseries view"
}
),
400,
)
valid_cols = set(column_types.keys())
valid_cols.update(params.derived_columns.keys())
if params.graph_type == "timeseries":
if params.x_axis is None:
for cand in ["time", "timestamp"]:
if cand in valid_cols:
params.x_axis = cand
break
if params.x_axis is None or params.x_axis not in valid_cols:
return jsonify({"error": "Invalid x_axis"}), 400
ctype = column_types.get(params.x_axis, "").upper()
if not any(t in ctype for t in ["TIMESTAMP", "DATE", "TIME"]):
return jsonify({"error": "x_axis must be a time column"}), 400
for col in params.columns:
if col not in valid_cols:
return jsonify({"error": f"Unknown column: {col}"}), 400
for col in params.group_by:
if col not in valid_cols:
return jsonify({"error": f"Unknown column: {col}"}), 400
if params.order_by and params.order_by not in valid_cols:
return jsonify({"error": f"Unknown column: {params.order_by}"}), 400
if params.group_by or params.graph_type == "timeseries":
agg = (params.aggregate or "avg").lower()
if agg.startswith("p") or agg == "sum":
need_numeric = True
allow_time = False
elif agg == "avg" or agg in {"min", "max"}:
need_numeric = False
allow_time = True
else:
need_numeric = False
allow_time = False
if need_numeric or allow_time:
for c in params.columns:
if c in params.group_by or c == params.x_axis:
continue
if c not in column_types:
continue
ctype = column_types.get(c, "").upper()
is_numeric = any(
t in ctype
for t in [
"INT",
"DECIMAL",
"REAL",
"DOUBLE",
"FLOAT",
"NUMERIC",
"HUGEINT",
]
)
is_time = "TIMESTAMP" in ctype or "DATE" in ctype or "TIME"
in ctype
if need_numeric and not is_numeric:
return (
jsonify(
{
"error": f"Aggregate {agg} cannot be applied
to column {c}",
}
),
400,
)
if allow_time and not (is_numeric or is_time):
return (
jsonify(
{
"error": f"Aggregate {agg} cannot be applied
to column {c}",
}
),
400,
)
if params.start is None or params.end is None:
axis = params.x_axis or "timestamp"
row = cast(
tuple[datetime | None, datetime | None],
con.execute(f"SELECT min({axis}), max({axis}) FROM events").fetc
hall()[
0
],
)
mn, mx = row
if params.start is None and mn is not None:
params.start = (
mn.strftime("%Y-%m-%d %H:%M:%S") if not isinstance(mn, str)
else mn
)
if params.end is None and mx is not None:
params.end = (
mx.strftime("%Y-%m-%d %H:%M:%S") if not isinstance(mx, str)
else mx
)
bucket_size: int | None = None
if params.graph_type == "timeseries":
bucket_size = _granularity_seconds(
params.granularity,
params.start if isinstance(params.start, str) else None,
params.end if isinstance(params.end, str) else None,
)root@3fe4fa1f8226:/workspace/scubaduck# sed -n '460,540p' scubaduck/server.py
sql = build_query(params, column_types)
try:
rows = con.execute(sql).fetchall()
except Exception as exc:
tb = traceback.format_exc()
print(f"Query failed:\n{sql}\n{tb}")
return (
jsonify({"sql": sql, "error": str(exc), "traceback": tb}),
400,
)
result: Dict[str, Any] = {"sql": sql, "rows": rows}
if params.start is not None:
result["start"] = str(params.start)
if params.end is not None:
result["end"] = str(params.end)
if bucket_size is not None:
result["bucket_size"] = bucket_size
return jsonify(result)
return app
app = create_app()
if __name__ == "__main__":
app.run(debug=True)I'll cache column types by table rather than globally, updating per request with table param.
I’ll adjust build_query to compute column_types based on the table in params.
@dataclass
class Filter:
column: str
op: str
value: str | int | float | list[str] | None
@dataclass
class QueryParams:
start: str | None = None
end: str | None = None
order_by: str | None = None
order_dir: str = "ASC"
limit: int | None = None
columns: list[str] = field(default_factory=lambda: [])
filters: list[Filter] = field(default_factory=lambda: [])
derived_columns: dict[str, str] = field(default_factory=lambda: {})
graph_type: str = "samples"
group_by: list[str] = field(default_factory=lambda: [])
aggregate: str | None = None
show_hits: bool = False
x_axis: str | None = None
granularity: str = "Auto"
fill: str = "0"
def _load_database(path: Path) -> duckdb.DuckDBPyConnection:
if not path.exists():
raise FileNotFoundError(path)
ext = path.suffix.lower()assert "events" in header
assert page.is_visible("#settings")
assert page.is_hidden("#columns")
page.click("text=Columns")
assert page.is_visible("#columns")
cols = [c.strip() for c in page.locator("#column_groups li").all_inner_texts
()]
assert "timestamp" in cols
assert "event" in cols
page.click("text=View Settings")
assert page.is_visible("#settings")
btn_color = page.evaluate(
"getComputedStyle(document.querySelector('#dive')).backgroundColor"
)
assert "rgb(0, 128, 0)" == btn_color
sidebar_overflow = page.evaluate(
"getComputedStyle(document.querySelector('#sidebar')).overflowY"
)
view_overflow = page.evaluate(I'll scroll between lines 170 and 200 to find what I need.
view_overflow = page.evaluate(
"getComputedStyle(document.querySelector('#view')).overflowY"
)
assert sidebar_overflow == "auto"
assert view_overflow == "auto"
def test_graph_type_table_fields(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#graph_type", state="attached")
select_value(page, "#graph_type", "table")
assert page.is_visible("#group_by_field")
assert page.is_visible("#aggregate_field")
assert page.is_visible("#show_hits_field")
page.click("text=Columns")
assert not page.is_visible("text=Strings:")
def test_graph_type_timeseries_fields(page: Any, server_url: str) -> None:
page.goto(server_url)
page.wait_for_selector("#graph_type", state="attached")
select_value(page, "#graph_type", "timeseries")
assert page.is_visible("#group_by_field")
assert page.is_visible("#aggregate_field")
assert page.is_visible("#x_axis_field")
assert page.is_visible("#granularity_field")
assert page.is_visible("#fill_field")
def test_timeseries_default_query(page: Any, server_url: str) -> None:
page.goto(server_url)I'll update the header_and_tabs test to check for sample.csv, and table selects. Need to find where server_url is defined—I'll search server_url in tests/test_web.py.
y | head
54:def test_range_filters(page: Any, server_url: str) -> None:
57: server_url,
73:def test_order_by(page: Any, server_url: str) -> None:
76: server_url,
87:def test_limit(page: Any, server_url: str) -> None:
90: server_url,
99:def test_simple_filter(page: Any, server_url: str) -> None:
100: page.goto(server_url)
120:def test_default_filter_and_layout(page: Any, server_url: str) -> None:
121: page.goto(server_url)I'll check the beginning of tests/test_web.py to understand the environment setup.
from __future__ import annotations
from typing import Any
def select_value(page: Any, selector: str, value: str) -> None:
page.evaluate(
"arg => setSelectValue(arg.sel, arg.val)",
{"sel": selector, "val": value},
)
def run_query(
page: Any,
url: str,It looks like there’s no server_url fixture. Let me check tests/conftest.py.
root@3fe4fa1f8226:/workspace/scubaduck# ls tests
conftest.py test_server.py test_web.pyimport threading
from collections.abc import Iterator
import pytest
from werkzeug.serving import make_server
from scubaduck.server import app
@pytest.fixture()
def server_url() -> Iterator[str]:
httpd = make_server("127.0.0.1", 0, app)
port = httpd.server_port
thread = threading.Thread(target=httpd.serve_forever)
thread.start()
try:
yield f"http://127.0.0.1:{port}"
finally:
httpd.shutdown()
thread.join()Time to tweak server.py: add features, update database loading, table handling, and routes.
Adding table parameter to functions, modifying caches, and implementing /api/tables for frontend integration.
Update header, CSS, and tests to match table selection without auto-running the query.
Setting up table API tests and verifying caching, quoting, and query building with params.table.
root@3fe4fa1f8226:/workspace/scubaduck# grep -n "quote_identifier" -n scubaduck/
server.pyI’ll start by activating the virtual environment and checking the pyproject configuration.
[project]
name = "scubaduck"
version = "0.1.0"
description = "Scuba on DuckDB"
readme = "README.md"
requires-python = ">=3.12"
license = {text = "MIT"}
dependencies = [
"duckdb",
"flask",
"python-dateutil>=2.9.0.post0",
]
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"
[tool.pytest.ini_options]
addopts = "-n auto --tb=native"
[tool.pyright]
# Pyright configuration with strict settings
include = ["scubaduck", "tests"]
exclude = ["**/__pycache__", "dist"]
stubPath = "stubs"
venvPath = "."
venv = ".venv"
reportMissingImports = true
reportMissingTypeStubs = true
pythonVersion = "3.12"
pythonPlatform = "All"
typeCheckingMode = "strict"
reportUnknownMemberType = true
reportUnknownParameterType = true
reportUnknownVariableType = true
reportUnknownArgumentType = true
reportPrivateImportUsage = true
reportUntypedFunctionDecorator = true
reportFunctionMemberAccess = true
reportIncompatibleMethodOverride = true
[tool.ruff]
target-version = "py312"
[tool.uv]
dev-dependencies = [
"pytest>=8.3.5",
"ruff>=0.11.2",
"pytest-xdist>=3.6.1",
"playwright>=1.52.0",
"pyright>=1.1.350",
"pytest-playwright>=0.7.0",
]root@3fe4fa1f8226:/workspace/scubaduck# source .venv/bin/activate
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I'm going to check the status of the git repository now.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# git status --short(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# apply_patch<<'PATCH'
> *** Begin Patch
> *** Update File: scubaduck/server.py
> @@
> class QueryParams:
> @@
> fill: str = "0"
> + table: str = "events"
> *** End Patch
> PATCH
Success. Updated the following files:
M scubaduck/server.py
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I'll make changes to the _load_database function now.
rver.py
ext = path.suffix.lower()
if ext == ".csv":
con = duckdb.connect()
con.execute(
f"CREATE TABLE events AS SELECT * FROM read_csv_auto('{path.as_posix
()}')"
)
elif ext in {".db", ".sqlite"}:
con = duckdb.connect()
sconn = sqlite3.connect(path)
info = sconn.execute("PRAGMA table_info(events)").fetchall()
col_defs = ", ".join(f"{r[1]} {r[2]}" for r in info)
con.execute(f"CREATE TABLE events ({col_defs})")
placeholders = ",".join("?" for _ in info)
for row in sconn.execute("SELECT * FROM events"):
con.execute(f"INSERT INTO events VALUES ({placeholders})", row)
sconn.close()
else:
con = duckdb.connect(path)
return con
_REL_RE = re.compile(
r"([+-]?\d+(?:\.\d*)?)\s*(hour|hours|day|days|week|weeks|fortnight|fortnight
s)",
re.IGNORECASE,
)
def parse_time(val: str | None) -> str | None:
"""Parse an absolute or relative time string into ``YYYY-MM-DD HH:MM:SS``.""
"
if val is None or val == "":
return None
s = val.strip()
if s.lower() == "now":
dt = datetime.now(timezone.utc)
return dt.replace(microsecond=0).strftime("%Y-%m-%d %H:%M:%S")
m = _REL_RE.fullmatch(s)
if m:
qty = float(m.group(1))
unit = m.group(2).lower()
delta: timedelta
if unit.startswith("hour"):
delta = timedelta(hours=qty)
elif unit.startswith("day"):
delta = timedelta(days=qty)
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I'm planning to modify the _load_database function now. Let's dive into that.
> *** Begin Patch
> *** Update File: scubaduck/server.py
> @@
> - ext = path.suffix.lower()
> - if ext == ".csv":
> - con = duckdb.connect()
> - con.execute(
> - f"CREATE TABLE events AS SELECT * FROM read_csv_auto('{path.as_po
six()}')"
> - )
> - elif ext in {".db", ".sqlite"}:
> - con = duckdb.connect()
> - sconn = sqlite3.connect(path)
> - info = sconn.execute("PRAGMA table_info(events)").fetchall()
> - col_defs = ", ".join(f"{r[1]} {r[2]}" for r in info)
> - con.execute(f"CREATE TABLE events ({col_defs})")
> - placeholders = ",".join("?" for _ in info)
> - for row in sconn.execute("SELECT * FROM events"):
> - con.execute(f"INSERT INTO events VALUES ({placeholders})", row)
> - sconn.close()
> - else:
> - con = duckdb.connect(path)
> + ext = path.suffix.lower()
> + if ext == ".csv":
> + con = duckdb.connect()
> + con.execute(
> + f"CREATE TABLE events AS SELECT * FROM read_csv_auto('{path.as_po
six()}')"
> + )
> + elif ext in {".db", ".sqlite"}:
> + con = duckdb.connect()
> + con.execute("INSTALL sqlite")
> + con.execute("LOAD sqlite")
> + con.execute(f"ATTACH '{path.as_posix()}' AS db (TYPE SQLITE)")
> + tables = [r[0] for r in con.execute("SELECT name FROM db.sqlite_maste
r WHERE type='table'").fetchall()]
> + for t in tables:
> + con.execute(f'CREATE VIEW "{t}" AS SELECT * FROM db."{t}"')
> + else:
> + con = duckdb.connect(path)
> *** End Patch
> PATCH
Success. Updated the following files:
M scubaduck/server.py
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I've added SQLite to DuckDB. Now, I'll handle table listing and column caching.
server.py
return query
def create_app(db_file: str | Path | None = None) -> Flask:
app = Flask(__name__, static_folder="static")
if db_file is None:
env_db = os.environ.get("SCUBADUCK_DB")
if env_db:
db_file = env_db
db_path = Path(db_file or Path(__file__).with_name("sample.csv")).resolve()
con = _load_database(db_path)
column_types: Dict[str, str] = {
r[1]: r[2] for r in con.execute("PRAGMA table_info(events)").fetchall()
}
sample_cache: Dict[Tuple[str, str], Tuple[List[str], float]] = {}
CACHE_TTL = 60.0
CACHE_LIMIT = 200
@app.route("/")
def index() -> Any: # pyright: ignore[reportUnusedFunction]
assert app.static_folder is not None
return send_from_directory(app.static_folder, "index.html")
@app.route("/js/<path:filename>")
def js(filename: str) -> Any: # pyright: ignore[reportUnusedFunction]
assert app.static_folder is not None
folder = Path(app.static_folder) / "js"
return send_from_directory(folder, filename)
@app.route("/api/columns")
def columns() -> Any: # pyright: ignore[reportUnusedFunction]
rows = con.execute("PRAGMA table_info(events)").fetchall()
return jsonify([{"name": r[1], "type": r[2]} for r in rows])
def _cache_get(key: Tuple[str, str]) -> List[str] | None:
item = sample_cache.get(key)
if item is None:
return None
vals, ts = item
if time.time() - ts > CACHE_TTL:
del sample_cache[key]
return None
sample_cache[key] = (vals, time.time())
return vals
def _cache_set(key: Tuple[str, str], vals: List[str]) -> None:
sample_cache[key] = (vals, time.time())
if len(sample_cache) > CACHE_LIMIT:
oldest = min(sample_cache.items(), key=lambda kv: kv[1][1])[0]
del sample_cache[oldest]
@app.route("/api/samples")
def sample_values() -> Any: # pyright: ignore[reportUnusedFunction]
column = request.args.get("column")
substr = request.args.get("q", "")
if not column or column not in column_types:
return jsonify([])
ctype = column_types[column].upper()
if "CHAR" not in ctype and "STRING" not in ctype and "VARCHAR" not in ct
ype:
return jsonify([])
key = (column, substr)
cached = _cache_get(key)
if cached is not None:
return jsonify(cached)
rows = con.execute(
f"SELECT DISTINCT {column} FROM events WHERE CAST({column} AS VARCHA
R) ILIKE '%' || ? || '%' LIMIT 20",
[substr],
).fetchall()
values = [r[0] for r in rows]
_cache_set(key, values)
return jsonify(values)
@app.route("/api/query", methods=["POST"])
def query() -> Any: # pyright: ignore[reportUnusedFunction]
payload = request.get_json(force=True)
try:
start = parse_time(payload.get("start"))
end = parse_time(payload.get("end"))
except Exception as exc:
return jsonify({"error": str(exc)}), 400
params = QueryParams(
start=start,
end=end,
order_by=payload.get("order_by"),
order_dir=payload.get("order_dir", "ASC"),
limit=payload.get("limit"),
columns=payload.get("columns", []),
derived_columns=payload.get("derived_columns", {}),
graph_type=payload.get("graph_type", "samples"),
group_by=payload.get("group_by", []),
aggregate=payload.get("aggregate"),
show_hits=payload.get("show_hits", False),
x_axis=payload.get("x_axis"),
granularity=payload.get("granularity", "Auto"),
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I'll store tables like the default_table and column_types_cache, then proceed with the patch.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# apply_patch<<'PATCH'
> *** Begin Patch
> *** Update File: scubaduck/server.py
> @@
> - con = _load_database(db_path)
> - column_types: Dict[str, str] = {
> - r[1]: r[2] for r in con.execute("PRAGMA table_info(events)").fetchall
()
> - }
> + con = _load_database(db_path)
> + tables = [r[0] for r in con.execute("SHOW TABLES").fetchall()]
> + if not tables:
> + raise ValueError("No tables found in database")
> + default_table = tables[0]
> + columns_cache: Dict[str, Dict[str, str]] = {}
> +
> + def get_columns(table: str) -> Dict[str, str]:
> + if table not in columns_cache:
> + rows = con.execute(f'PRAGMA table_info("{table}")').fetchall()
> + if not rows:
> + raise ValueError(f"Unknown table: {table}")
> + columns_cache[table] = {r[1]: r[2] for r in rows}
> + return columns_cache[table]
> *** End Patch
> PATCH
Success. Updated the following files:
M scubaduck/server.py
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I'll update the sample_cache and related elements accordingly.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# sed -n '253,345p' scubaduck/
server.py
def create_app(db_file: str | Path | None = None) -> Flask:
app = Flask(__name__, static_folder="static")
if db_file is None:
env_db = os.environ.get("SCUBADUCK_DB")
if env_db:
db_file = env_db
db_path = Path(db_file or Path(__file__).with_name("sample.csv")).resolve()
con = _load_database(db_path)
tables = [r[0] for r in con.execute("SHOW TABLES").fetchall()]
if not tables:
raise ValueError("No tables found in database")
default_table = tables[0]
columns_cache: Dict[str, Dict[str, str]] = {}
def get_columns(table: str) -> Dict[str, str]:
if table not in columns_cache:
rows = con.execute(f'PRAGMA table_info("{table}")').fetchall()
if not rows:
raise ValueError(f"Unknown table: {table}")
columns_cache[table] = {r[1]: r[2] for r in rows}
return columns_cache[table]
sample_cache: Dict[Tuple[str, str], Tuple[List[str], float]] = {}
CACHE_TTL = 60.0
CACHE_LIMIT = 200
@app.route("/")
def index() -> Any: # pyright: ignore[reportUnusedFunction]
assert app.static_folder is not None
return send_from_directory(app.static_folder, "index.html")
@app.route("/js/<path:filename>")
def js(filename: str) -> Any: # pyright: ignore[reportUnusedFunction]
assert app.static_folder is not None
folder = Path(app.static_folder) / "js"
return send_from_directory(folder, filename)
@app.route("/api/columns")
def columns() -> Any: # pyright: ignore[reportUnusedFunction]
rows = con.execute("PRAGMA table_info(events)").fetchall()
return jsonify([{"name": r[1], "type": r[2]} for r in rows])
def _cache_get(key: Tuple[str, str]) -> List[str] | None:
item = sample_cache.get(key)
if item is None:
return None
vals, ts = item
if time.time() - ts > CACHE_TTL:
del sample_cache[key]
return None
sample_cache[key] = (vals, time.time())
return vals
def _cache_set(key: Tuple[str, str], vals: List[str]) -> None:
sample_cache[key] = (vals, time.time())
if len(sample_cache) > CACHE_LIMIT:
oldest = min(sample_cache.items(), key=lambda kv: kv[1][1])[0]
del sample_cache[oldest]
@app.route("/api/samples")
def sample_values() -> Any: # pyright: ignore[reportUnusedFunction]
column = request.args.get("column")
substr = request.args.get("q", "")
if not column or column not in column_types:
return jsonify([])
ctype = column_types[column].upper()
if "CHAR" not in ctype and "STRING" not in ctype and "VARCHAR" not in ct
ype:
return jsonify([])
key = (column, substr)
cached = _cache_get(key)
if cached is not None:
return jsonify(cached)
rows = con.execute(
f"SELECT DISTINCT {column} FROM events WHERE CAST({column} AS VARCHA
R) ILIKE '%' || ? || '%' LIMIT 20",
[substr],
).fetchall()
values = [r[0] for r in rows]
_cache_set(key, values)
return jsonify(values)
@app.route("/api/query", methods=["POST"])
def query() -> Any: # pyright: ignore[reportUnusedFunction]
payload = request.get_json(force=True)
try:
start = parse_time(payload.get("start"))
end = parse_time(payload.get("end"))
except Exception as exc:
return jsonify({"error": str(exc)}), 400
params = QueryParams(
start=start,
end=end,
order_by=payload.get("order_by"),
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I'll adjust the column types and make necessary line updates.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# apply_patch<<'PATCH'
> *** Begin Patch
> *** Update File: scubaduck/server.py
> @@
> - sample_cache: Dict[Tuple[str, str], Tuple[List[str], float]] = {}
> + sample_cache: Dict[Tuple[str, str, str], Tuple[List[str], float]] = {}
> @@
> - @app.route("/api/columns")
> - def columns() -> Any: # pyright: ignore[reportUnusedFunction]
> - rows = con.execute("PRAGMA table_info(events)").fetchall()
> - return jsonify([{"name": r[1], "type": r[2]} for r in rows])
> + @app.route("/api/tables")
> + def tables_endpoint() -> Any: # pyright: ignore[reportUnusedFunction]
> + return jsonify(tables)
> +
> + @app.route("/api/columns")
> + def columns() -> Any: # pyright: ignore[reportUnusedFunction]
> + table = request.args.get("table", default_table)
> + rows = con.execute(f'PRAGMA table_info("{table}")').fetchall()
> + return jsonify([{"name": r[1], "type": r[2]} for r in rows])
> *** End Patch
> PATCH
Success. Updated the following files:
M scubaduck/server.py
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I'll update the cache helper functions to properly utilize the table parameter.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# sed -n '310,381p' scubaduck/
server.py
def _cache_set(key: Tuple[str, str], vals: List[str]) -> None:
sample_cache[key] = (vals, time.time())
if len(sample_cache) > CACHE_LIMIT:
oldest = min(sample_cache.items(), key=lambda kv: kv[1][1])[0]
del sample_cache[oldest]
@app.route("/api/samples")
def sample_values() -> Any: # pyright: ignore[reportUnusedFunction]
column = request.args.get("column")
substr = request.args.get("q", "")
if not column or column not in column_types:
return jsonify([])
ctype = column_types[column].upper()
if "CHAR" not in ctype and "STRING" not in ctype and "VARCHAR" not in ct
ype:
return jsonify([])
key = (column, substr)
cached = _cache_get(key)
if cached is not None:
return jsonify(cached)
rows = con.execute(
f"SELECT DISTINCT {column} FROM events WHERE CAST({column} AS VARCHA
R) ILIKE '%' || ? || '%' LIMIT 20",
[substr],
).fetchall()
values = [r[0] for r in rows]
_cache_set(key, values)
return jsonify(values)
@app.route("/api/query", methods=["POST"])
def query() -> Any: # pyright: ignore[reportUnusedFunction]
payload = request.get_json(force=True)
try:
start = parse_time(payload.get("start"))
end = parse_time(payload.get("end"))
except Exception as exc:
return jsonify({"error": str(exc)}), 400
params = QueryParams(
start=start,
end=end,
order_by=payload.get("order_by"),
order_dir=payload.get("order_dir", "ASC"),
limit=payload.get("limit"),
columns=payload.get("columns", []),
derived_columns=payload.get("derived_columns", {}),
graph_type=payload.get("graph_type", "samples"),
group_by=payload.get("group_by", []),
aggregate=payload.get("aggregate"),
show_hits=payload.get("show_hits", False),
x_axis=payload.get("x_axis"),
granularity=payload.get("granularity", "Auto"),
fill=payload.get("fill", "0"),
)
for f in payload.get("filters", []):
params.filters.append(Filter(f["column"], f["op"], f.get("value")))
if params.graph_type not in {"table", "timeseries"} and (
params.group_by or params.aggregate or params.show_hits
):
return (
jsonify(
{
"error": "group_by, aggregate and show_hits are only val
id for table or timeseries view"
}
),
400,
)
valid_cols = set(column_types.keys())
valid_cols.update(params.derived_columns.keys())
if params.graph_type == "timeseries":
if params.x_axis is None:
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I’ll go ahead and patch the parts that need updates.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# apply_patch<<'PATCH'
> *** Begin Patch
> *** Update File: scubaduck/server.py
> @@
> - def _cache_get(key: Tuple[str, str]) -> List[str] | None:
> + def _cache_get(key: Tuple[str, str, str]) -> List[str] | None:
> @@
> - def _cache_set(key: Tuple[str, str], vals: List[str]) -> None:
> + def _cache_set(key: Tuple[str, str, str], vals: List[str]) -> None:
> @@
> - def sample_values() -> Any: # pyright: ignore[reportUnusedFunction]
> - column = request.args.get("column")
> - substr = request.args.get("q", "")
> - if not column or column not in column_types:
> - return jsonify([])
> - ctype = column_types[column].upper()
> + def sample_values() -> Any: # pyright: ignore[reportUnusedFunction]
> + table = request.args.get("table", default_table)
> + column = request.args.get("column")
> + substr = request.args.get("q", "")
> + column_types = get_columns(table)
> + if not column or column not in column_types:
> + return jsonify([])
> + ctype = column_types[column].upper()
> @@
> - key = (column, substr)
> + key = (table, column, substr)
> @@
> - rows = con.execute(
> - f"SELECT DISTINCT {column} FROM events WHERE CAST({column} AS VAR
CHAR) ILIKE '%' || ? || '%' LIMIT 20",
> - [substr],
> - ).fetchall()
> + rows = con.execute(
> + f'SELECT DISTINCT {column} FROM "{table}" WHERE CAST({column} AS
VARCHAR) ILIKE '%' || ? || '%' LIMIT 20',
> + [substr],
> + ).fetchall()
> *** End Patch
> PATCH
Success. Updated the following files:
M scubaduck/server.py
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#The f-string is tricky with quoting. I’ll fix it by using double-quotes for the f-string.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# grep -n "ILIKE" -n scubaduck
/server.py
231: where_parts.append(f"{f.column} ILIKE '%' || {val} || '%'")
233: where_parts.append(f"{f.column} NOT ILIKE '%' || {val} || '%'")
333: f'SELECT DISTINCT {column} FROM "{table}" WHERE CAST({column} AS
VARCHAR) ILIKE '%' || ? || '%' LIMIT 20',
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I'm checking around line 325 to locate what I'm looking for.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# sed -n '324,337p' scubaduck/
server.py
return jsonify([])
ctype = column_types[column].upper()
if "CHAR" not in ctype and "STRING" not in ctype and "VARCHAR" not in ct
ype:
return jsonify([])
key = (table, column, substr)
cached = _cache_get(key)
if cached is not None:
return jsonify(cached)
rows = con.execute(
f'SELECT DISTINCT {column} FROM "{table}" WHERE CAST({column} AS VAR
CHAR) ILIKE '%' || ? || '%' LIMIT 20',
[substr],
).fetchall()
values = [r[0] for r in rows]
_cache_set(key, values)
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#The f-string had an incorrect quote inside, so I’ll fix it with double quotes outside.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# apply_patch<<'PATCH'
> *** Begin Patch
> *** Update File: scubaduck/server.py
> @@
> - rows = con.execute(
> - f'SELECT DISTINCT {column} FROM "{table}" WHERE CAST({column} AS
VARCHAR) ILIKE '%' || ? || '%' LIMIT 20',
> - [substr],
> - ).fetchall()
> + rows = con.execute(
> + f"SELECT DISTINCT {column} FROM \"{table}\" WHERE CAST({column} A
S VARCHAR) ILIKE '%' || ? || '%' LIMIT 20",
> + [substr],
> + ).fetchall()
> *** End Patch
> PATCH
Success. Updated the following files:
M scubaduck/server.py
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#Time to update the caching function definitions to align with the new requirements.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# sed -n '300,342p' scubaduck/
server.py
def _cache_get(key: Tuple[str, str, str]) -> List[str] | None:
item = sample_cache.get(key)
if item is None:
return None
vals, ts = item
if time.time() - ts > CACHE_TTL:
del sample_cache[key]
return None
sample_cache[key] = (vals, time.time())
return vals
def _cache_set(key: Tuple[str, str, str], vals: List[str]) -> None:
sample_cache[key] = (vals, time.time())
if len(sample_cache) > CACHE_LIMIT:
oldest = min(sample_cache.items(), key=lambda kv: kv[1][1])[0]
del sample_cache[oldest]
@app.route("/api/samples")
def sample_values() -> Any: # pyright: ignore[reportUnusedFunction]
table = request.args.get("table", default_table)
column = request.args.get("column")
substr = request.args.get("q", "")
column_types = get_columns(table)
if not column or column not in column_types:
return jsonify([])
ctype = column_types[column].upper()
if "CHAR" not in ctype and "STRING" not in ctype and "VARCHAR" not in ct
ype:
return jsonify([])
key = (table, column, substr)
cached = _cache_get(key)
if cached is not None:
return jsonify(cached)
rows = con.execute(
f"SELECT DISTINCT {column} FROM \"{table}\" WHERE CAST({column} AS V
ARCHAR) ILIKE '%' || ? || '%' LIMIT 20",
[substr],
).fetchall()
values = [r[0] for r in rows]
_cache_set(key, values)
return jsonify(values)
@app.route("/api/query", methods=["POST"])
def query() -> Any: # pyright: ignore[reportUnusedFunction]
payload = request.get_json(force=True)
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#Time to update the Query endpoint now. Let's dive into the changes!
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# sed -n '342,464p' scubaduck/
server.py
payload = request.get_json(force=True)
try:
start = parse_time(payload.get("start"))
end = parse_time(payload.get("end"))
except Exception as exc:
return jsonify({"error": str(exc)}), 400
params = QueryParams(
start=start,
end=end,
order_by=payload.get("order_by"),
order_dir=payload.get("order_dir", "ASC"),
limit=payload.get("limit"),
columns=payload.get("columns", []),
derived_columns=payload.get("derived_columns", {}),
graph_type=payload.get("graph_type", "samples"),
group_by=payload.get("group_by", []),
aggregate=payload.get("aggregate"),
show_hits=payload.get("show_hits", False),
x_axis=payload.get("x_axis"),
granularity=payload.get("granularity", "Auto"),
fill=payload.get("fill", "0"),
)
for f in payload.get("filters", []):
params.filters.append(Filter(f["column"], f["op"], f.get("value")))
if params.graph_type not in {"table", "timeseries"} and (
params.group_by or params.aggregate or params.show_hits
):
return (
jsonify(
{
"error": "group_by, aggregate and show_hits are only val
id for table or timeseries view"
}
),
400,
)
valid_cols = set(column_types.keys())
valid_cols.update(params.derived_columns.keys())
if params.graph_type == "timeseries":
if params.x_axis is None:
for cand in ["time", "timestamp"]:
if cand in valid_cols:
params.x_axis = cand
break
if params.x_axis is None or params.x_axis not in valid_cols:
return jsonify({"error": "Invalid x_axis"}), 400
ctype = column_types.get(params.x_axis, "").upper()
if not any(t in ctype for t in ["TIMESTAMP", "DATE", "TIME"]):
return jsonify({"error": "x_axis must be a time column"}), 400
for col in params.columns:
if col not in valid_cols:
return jsonify({"error": f"Unknown column: {col}"}), 400
for col in params.group_by:
if col not in valid_cols:
return jsonify({"error": f"Unknown column: {col}"}), 400
if params.order_by and params.order_by not in valid_cols:
return jsonify({"error": f"Unknown column: {params.order_by}"}), 400
if params.group_by or params.graph_type == "timeseries":
agg = (params.aggregate or "avg").lower()
if agg.startswith("p") or agg == "sum":
need_numeric = True
allow_time = False
elif agg == "avg" or agg in {"min", "max"}:
need_numeric = False
allow_time = True
else:
need_numeric = False
allow_time = False
if need_numeric or allow_time:
for c in params.columns:
if c in params.group_by or c == params.x_axis:
continue
if c not in column_types:
continue
ctype = column_types.get(c, "").upper()
is_numeric = any(
t in ctype
for t in [
"INT",
"DECIMAL",
"REAL",
"DOUBLE",
"FLOAT",
"NUMERIC",
"HUGEINT",
]
)
is_time = "TIMESTAMP" in ctype or "DATE" in ctype or "TIME"
in ctype
if need_numeric and not is_numeric:
return (
jsonify(
{
"error": f"Aggregate {agg} cannot be applied
to column {c}",
}
),
400,
)
if allow_time and not (is_numeric or is_time):
return (
jsonify(
{
"error": f"Aggregate {agg} cannot be applied
to column {c}",
}
),
400,
)
if params.start is None or params.end is None:
axis = params.x_axis or "timestamp"
row = cast(
tuple[datetime | None, datetime | None],
con.execute(f"SELECT min({axis}), max({axis}) FROM events").fetc
hall()[
0
],
)
mn, mx = row
if params.start is None and mn is not None:
params.start = (
mn.strftime("%Y-%m-%d %H:%M:%S") if not isinstance(mn, str)
else mn
)
if params.end is None and mx is not None:
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I'll parse the table from the payload and add it to QueryParams before computing column_types.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# apply_patch<<'PATCH'
> *** Begin Patch
> *** Update File: scubaduck/server.py
> @@
> params = QueryParams(
> start=start,
> end=end,
> order_by=payload.get("order_by"),
> order_dir=payload.get("order_dir", "ASC"),
> limit=payload.get("limit"),
> columns=payload.get("columns", []),
> derived_columns=payload.get("derived_columns", {}),
> graph_type=payload.get("graph_type", "samples"),
> group_by=payload.get("group_by", []),
> aggregate=payload.get("aggregate"),
> show_hits=payload.get("show_hits", False),
> x_axis=payload.get("x_axis"),
> granularity=payload.get("granularity", "Auto"),
> - fill=payload.get("fill", "0"),
> + fill=payload.get("fill", "0"),
> + table=payload.get("table", default_table),
> )
> *** End Patch
> PATCH
Success. Updated the following files:
M scubaduck/server.py
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I’ll start by using column_types = get_columns(params.table) and continue patching.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# grep -n "valid_cols" -n scub
aduck/server.py
381: valid_cols = set(column_types.keys())
382: valid_cols.update(params.derived_columns.keys())
386: if cand in valid_cols:
389: if params.x_axis is None or params.x_axis not in valid_cols:
395: if col not in valid_cols:
398: if col not in valid_cols:
400: if params.order_by and params.order_by not in valid_cols:
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I’ll change column_types to use get_columns(params.table) within the query function.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# sed -n '355,390p' scubaduck/
server.py
columns=payload.get("columns", []),
derived_columns=payload.get("derived_columns", {}),
graph_type=payload.get("graph_type", "samples"),
group_by=payload.get("group_by", []),
aggregate=payload.get("aggregate"),
show_hits=payload.get("show_hits", False),
x_axis=payload.get("x_axis"),
granularity=payload.get("granularity", "Auto"),
fill=payload.get("fill", "0"),
table=payload.get("table", default_table),
)
for f in payload.get("filters", []):
params.filters.append(Filter(f["column"], f["op"], f.get("value")))
if params.graph_type not in {"table", "timeseries"} and (
params.group_by or params.aggregate or params.show_hits
):
return (
jsonify(
{
"error": "group_by, aggregate and show_hits are only val
id for table or timeseries view"
}
),
400,
)
valid_cols = set(column_types.keys())
valid_cols.update(params.derived_columns.keys())
if params.graph_type == "timeseries":
if params.x_axis is None:
for cand in ["time", "timestamp"]:
if cand in valid_cols:
params.x_axis = cand
break
if params.x_axis is None or params.x_axis not in valid_cols:
return jsonify({"error": "Invalid x_axis"}), 400
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I need to add column_types = get_columns(params.table) right before valid_cols.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# apply_patch<<'PATCH'
> *** Begin Patch
> *** Update File: scubaduck/server.py
> @@
> - for f in payload.get("filters", []):
> - params.filters.append(Filter(f["column"], f["op"], f.get("value")
))
> + for f in payload.get("filters", []):
> + params.filters.append(Filter(f["column"], f["op"], f.get("value")
))
> +
> + column_types = get_columns(params.table)
> *** End Patch
> PATCH
Success. Updated the following files:
M scubaduck/server.py
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I'll update the queries to use params.table instead of the hardcoded values.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# grep -n "FROM events" -n scu
baduck/server.py
208: query = f"SELECT {select_clause} FROM events"
458: con.execute(f"SELECT min({axis}), max({axis}) FROM events").
fetchall()[
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I'll patch both the sample_values and earlier changes.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# sed -n '198,213p' scubaduck/
server.py
query += f" ORDER BY {params.order_by} {params.order_dir}"
elif params.graph_type == "timeseries":
query += " ORDER BY bucket"
if params.limit is not None:
query += f" LIMIT {params.limit}"
return query
for name, expr in params.derived_columns.items():
select_parts.append(f"{expr} AS {name}")
select_clause = ", ".join(select_parts) if select_parts else "*"
query = f"SELECT {select_clause} FROM events"
where_parts: list[str] = []
if params.start:
where_parts.append(f"timestamp >= '{params.start}'")
if params.end:
where_parts.append(f"timestamp <= '{params.end}'")
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I'll replace "FROM events" to reference the table instead.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# apply_patch<<'PATCH'
> *** Begin Patch
> *** Update File: scubaduck/server.py
> @@
> - query = f"SELECT {select_clause} FROM events"
> + query = f"SELECT {select_clause} FROM \"{params.table}\""
> *** End Patch
> PATCH
Success. Updated the following files:
M scubaduck/server.py
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I'll update other parts of the code where events are referenced.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# sed -n '446,470p' scubaduck/
server.py
return (
jsonify(
{
"error": f"Aggregate {agg} cannot be applied
to column {c}",
}
),
400,
)
if params.start is None or params.end is None:
axis = params.x_axis or "timestamp"
row = cast(
tuple[datetime | None, datetime | None],
con.execute(f"SELECT min({axis}), max({axis}) FROM events").fetc
hall()[
0
],
)
mn, mx = row
if params.start is None and mn is not None:
params.start = (
mn.strftime("%Y-%m-%d %H:%M:%S") if not isinstance(mn, str)
else mn
)
if params.end is None and mx is not None:
params.end = (
mx.strftime("%Y-%m-%d %H:%M:%S") if not isinstance(mx, str)
else mx
)
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I'll update the design to incorporate a table instead.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# apply_patch<<'PATCH'
> *** Begin Patch
> *** Update File: scubaduck/server.py
> @@
> - axis = params.x_axis or "timestamp"
> - row = cast(
> - tuple[datetime | None, datetime | None],
> - con.execute(f"SELECT min({axis}), max({axis}) FROM events").f
etchall()[
> - 0
> - ],
> - )
> + axis = params.x_axis or "timestamp"
> + row = cast(
> + tuple[datetime | None, datetime | None],
> + con.execute(f"SELECT min({axis}), max({axis}) FROM \"{params.
table}\"").fetchall()[0],
> + )
> *** End Patch
> PATCH
Success. Updated the following files:
M scubaduck/server.py
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I'll look for other events that might be relevant to the task.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# grep -n "events" -n scubaduc
k/server.py
44: table: str = "events"
55: f"CREATE TABLE events AS SELECT * FROM read_csv_auto('{path.as_po
six()}')"
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I think references could still appear after line 55, need to investigate further.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# grep -n "events" -n scubaduc
k/server.py | sed -n '1,120p'
44: table: str = "events"
55: f"CREATE TABLE events AS SELECT * FROM read_csv_auto('{path.as_po
six()}')"
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I'll check the "PRAGMA table_info(events)" and any other changes made earlier.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# grep -n "PRAGMA table_info"
-n scubaduck/server.py
269: rows = con.execute(f'PRAGMA table_info("{table}")').fetchall()
297: rows = con.execute(f'PRAGMA table_info("{table}")').fetchall()
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#Next, I'll patch the valid_cols and other references to the column_types variable.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# grep -n "column_types" -n sc
ubaduck/server.py
141:def build_query(params: QueryParams, column_types: Dict[str, str] | None = N
one) -> str:
167: if agg == "avg" and column_types is not None:
168: ctype = column_types.get(col, "").upper()
192: inner_sql = build_query(inner_params, column_types)
322: column_types = get_columns(table)
323: if not column or column not in column_types:
325: ctype = column_types[column].upper()
369: column_types = get_columns(params.table)
383: valid_cols = set(column_types.keys())
393: ctype = column_types.get(params.x_axis, "").upper()
420: if c not in column_types:
422: ctype = column_types.get(c, "").upper()
478: sql = build_query(params, column_types)
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I'll take a look around line 141 to check the relevant code.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# sed -n '140,210p' scubaduck/
server.py
def build_query(params: QueryParams, column_types: Dict[str, str] | None = None)
-> str:
select_parts: list[str] = []
group_cols = params.group_by[:]
if params.graph_type == "timeseries":
sec = _granularity_seconds(params.granularity, params.start, params.end)
x_axis = params.x_axis or "timestamp"
bucket_expr = (
f"TIMESTAMP 'epoch' + INTERVAL '{sec} second' * "
f"CAST(floor(epoch({x_axis})/{sec}) AS BIGINT)"
)
select_parts.append(f"{bucket_expr} AS bucket")
group_cols = ["bucket"] + group_cols
has_agg = bool(group_cols) or params.aggregate is not None
if has_agg:
select_cols = (
group_cols[1:] if params.graph_type == "timeseries" else group_cols
)
select_parts.extend(select_cols)
agg = (params.aggregate or "avg").lower()
def agg_expr(col: str) -> str:
if agg.startswith("p"):
quant = float(agg[1:]) / 100
return f"quantile({col}, {quant})"
if agg == "count distinct":
return f"count(DISTINCT {col})"
if agg == "avg" and column_types is not None:
ctype = column_types.get(col, "").upper()
if "TIMESTAMP" in ctype or "DATE" in ctype or "TIME" in ctype:
return (
"TIMESTAMP 'epoch' + INTERVAL '1 second' * "
f"CAST(avg(epoch({col})) AS BIGINT)"
)
return f"{agg}({col})"
for col in params.columns:
if col in group_cols:
continue
select_parts.append(f"{agg_expr(col)} AS {col}")
if params.show_hits:
select_parts.insert(len(group_cols), "count(*) AS Hits")
else:
select_parts.extend(params.columns)
if has_agg and params.derived_columns:
inner_params = replace(
params,
derived_columns={},
order_by=None,
limit=None,
)
inner_sql = build_query(inner_params, column_types)
outer_select = ["t.*"] + [
f"{expr} AS {name}" for name, expr in params.derived_columns.items()
]
query = f"SELECT {', '.join(outer_select)} FROM ({inner_sql}) t"
if params.order_by:
query += f" ORDER BY {params.order_by} {params.order_dir}"
elif params.graph_type == "timeseries":
query += " ORDER BY bucket"
if params.limit is not None:
query += f" LIMIT {params.limit}"
return query
for name, expr in params.derived_columns.items():
select_parts.append(f"{expr} AS {name}")
select_clause = ", ".join(select_parts) if select_parts else "*"
query = f"SELECT {select_clause} FROM \"{params.table}\""
where_parts: list[str] = []
if params.start:
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I'll grab the specific line ranges around 239 for the changes I made.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# grep -n "inner_sql" -n scuba
duck/server.py
192: inner_sql = build_query(inner_params, column_types)
196: query = f"SELECT {', '.join(outer_select)} FROM ({inner_sql}) t"
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I'll update the lines related to valid_cols and adjust for column_types.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# sed -n '380,460p' scubaduck/
server.py
400,
)
valid_cols = set(column_types.keys())
valid_cols.update(params.derived_columns.keys())
if params.graph_type == "timeseries":
if params.x_axis is None:
for cand in ["time", "timestamp"]:
if cand in valid_cols:
params.x_axis = cand
break
if params.x_axis is None or params.x_axis not in valid_cols:
return jsonify({"error": "Invalid x_axis"}), 400
ctype = column_types.get(params.x_axis, "").upper()
if not any(t in ctype for t in ["TIMESTAMP", "DATE", "TIME"]):
return jsonify({"error": "x_axis must be a time column"}), 400
for col in params.columns:
if col not in valid_cols:
return jsonify({"error": f"Unknown column: {col}"}), 400
for col in params.group_by:
if col not in valid_cols:
return jsonify({"error": f"Unknown column: {col}"}), 400
if params.order_by and params.order_by not in valid_cols:
return jsonify({"error": f"Unknown column: {params.order_by}"}), 400
if params.group_by or params.graph_type == "timeseries":
agg = (params.aggregate or "avg").lower()
if agg.startswith("p") or agg == "sum":
need_numeric = True
allow_time = False
elif agg == "avg" or agg in {"min", "max"}:
need_numeric = False
allow_time = True
else:
need_numeric = False
allow_time = False
if need_numeric or allow_time:
for c in params.columns:
if c in params.group_by or c == params.x_axis:
continue
if c not in column_types:
continue
ctype = column_types.get(c, "").upper()
is_numeric = any(
t in ctype
for t in [
"INT",
"DECIMAL",
"REAL",
"DOUBLE",
"FLOAT",
"NUMERIC",
"HUGEINT",
]
)
is_time = "TIMESTAMP" in ctype or "DATE" in ctype or "TIME"
in ctype
if need_numeric and not is_numeric:
return (
jsonify(
{
"error": f"Aggregate {agg} cannot be applied
to column {c}",
}
),
400,
)
if allow_time and not (is_numeric or is_time):
return (
jsonify(
{
"error": f"Aggregate {agg} cannot be applied
to column {c}",
}
),
400,
)
if params.start is None or params.end is None:
axis = params.x_axis or "timestamp"
row = cast(
tuple[datetime | None, datetime | None],
con.execute(f"SELECT min({axis}), max({axis}) FROM \"{params.tab
le}\"").fetchall()[0],
)
mn, mx = row
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I'll calculate the bucket size and finish by calling build_query.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# sed -n '460,520p' scubaduck/
server.py
mn, mx = row
if params.start is None and mn is not None:
params.start = (
mn.strftime("%Y-%m-%d %H:%M:%S") if not isinstance(mn, str)
else mn
)
if params.end is None and mx is not None:
params.end = (
mx.strftime("%Y-%m-%d %H:%M:%S") if not isinstance(mx, str)
else mx
)
bucket_size: int | None = None
if params.graph_type == "timeseries":
bucket_size = _granularity_seconds(
params.granularity,
params.start if isinstance(params.start, str) else None,
params.end if isinstance(params.end, str) else None,
)
sql = build_query(params, column_types)
try:
rows = con.execute(sql).fetchall()
except Exception as exc:
tb = traceback.format_exc()
print(f"Query failed:\n{sql}\n{tb}")
return (
jsonify({"sql": sql, "error": str(exc), "traceback": tb}),
400,
)
result: Dict[str, Any] = {"sql": sql, "rows": rows}
if params.start is not None:
result["start"] = str(params.start)
if params.end is not None:
result["end"] = str(params.end)
if bucket_size is not None:
result["bucket_size"] = bucket_size
return jsonify(result)
return app
app = create_app()
if __name__ == "__main__":
app.run(debug=True)
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I'll search for "column_types =" and remove it since it's replaced with get_columns.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# grep -n "column_types:" -n s
cubaduck/server.py
141:def build_query(params: QueryParams, column_types: Dict[str, str] | None = N
one) -> str:
323: if not column or column not in column_types:
420: if c not in column_types:
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#Looks like column_types should be local, but it's referenced at lines 150, 323, and 420.
ubaduck/server.py | head
141:def build_query(params: QueryParams, column_types: Dict[str, str] | None = N
one) -> str:
167: if agg == "avg" and column_types is not None:
168: ctype = column_types.get(col, "").upper()
192: inner_sql = build_query(inner_params, column_types)
322: column_types = get_columns(table)
323: if not column or column not in column_types:
325: ctype = column_types[column].upper()
369: column_types = get_columns(params.table)
383: valid_cols = set(column_types.keys())
393: ctype = column_types.get(params.x_axis, "").upper()
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I'll check lines around 320 to patch variables and ensure no leftovers.
server.py
@app.route("/api/samples")
def sample_values() -> Any: # pyright: ignore[reportUnusedFunction]
table = request.args.get("table", default_table)
column = request.args.get("column")
substr = request.args.get("q", "")
column_types = get_columns(table)
if not column or column not in column_types:
return jsonify([])
ctype = column_types[column].upper()
if "CHAR" not in ctype and "STRING" not in ctype and "VARCHAR" not in ct
ype:
return jsonify([])
key = (table, column, substr)
cached = _cache_get(key)
if cached is not None:
return jsonify(cached)
rows = con.execute(
f"SELECT DISTINCT {column} FROM \"{table}\" WHERE CAST({column} AS V
ARCHAR) ILIKE '%' || ? || '%' LIMIT 20",
[substr],
).fetchall()
values = [r[0] for r in rows]
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I'm adding an early check for invalid tables to prevent errors in query processing.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# apply_patch<<'PATCH'
> *** Begin Patch
> *** Update File: scubaduck/server.py
> @@
> params = QueryParams(
> @@
> fill=payload.get("fill", "0"),
> table=payload.get("table", default_table),
> )
> for f in payload.get("filters", []):
> params.filters.append(Filter(f["column"], f["op"], f.get("value")
))
> +
> + if params.table not in tables:
> + return jsonify({"error": "Invalid table"}), 400
>
> column_types = get_columns(params.table)
> *** End Patch
> PATCH
Success. Updated the following files:
M scubaduck/server.py
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I changed the caching key and annotation, then updated the query endpoint. Next, I’ll update the frontend HTML.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# grep -n "graph_type" -n scub
aduck/static/index.html | head
141: <div id="header">sample.csv - events <select id="graph_type"><option value
="samples">Samples</option><option value="table">Table</option><option value="ti
meseries">Time Series</option></select></div>
422:const graphTypeSel = document.getElementById('graph_type');
854: graph_type: graphTypeSel.value,
895: if (params.graph_type) sp.set('graph_type', params.graph_type);
896: if (params.graph_type === 'table' || params.graph_type === 'timeseries') {
901: if (params.graph_type === 'timeseries') {
921: graphTypeSel.value = params.graph_type || 'samples';
977: if (sp.has('graph_type')) params.graph_type = sp.get('graph_type');
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I'll work on modifying line 141 now.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# sed -n '138,144p' scubaduck/
static/index.html
</style>
</head>
<body>
<div id="header">sample.csv - events <select id="graph_type"><option value="sa
mples">Samples</option><option value="table">Table</option><option value="timese
ries">Time Series</option></select></div>
<div id="content">
<div id="sidebar">
<div id="tabs">
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I'll add a <select id="table"></select> to the code now.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# apply_patch<<'PATCH'
> *** Begin Patch
> *** Update File: scubaduck/static/index.html
> @@
> - <div id="header">sample.csv - events <select id="graph_type"><option value=
"samples">Samples</option><option value="table">Table</option><option value="tim
eseries">Time Series</option></select></div>
> + <div id="header">sample.csv - <select id="table"></select> <select id="grap
h_type"><option value="samples">Samples</option><option value="table">Table</opt
ion><option value="timeseries">Time Series</option></select></div>
> *** End Patch
> PATCH
Success. Updated the following files:
M scubaduck/static/index.html
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I'll look for #header or #table to possibly apply some CSS styling.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# grep -n "#header" -n scubadu
ck/static/index.html
8: #header { padding: 10px; font-weight: bold; border-bottom: 1px solid #ccc;
}
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I'll fetch tables and load columns by adding the tableSelect variable and adjusting the function.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# sed -n '288,312p' scubaduck/
static/index.html
<table id="results"></table>
</div>
</div>
<script src="/js/chip_input.js"></script>
<script src="/js/timeseries_chart.js"></script>
<script>
const allColumns = [];
const baseColumns = [];
const columnTypes = {};
const stringColumns = [];
const baseStringColumns = [];
const integerColumns = [];
const baseIntegerColumns = [];
const timeColumns = [];
const baseTimeColumns = [];
const derivedColumns = [];
let selectedColumns = [];
let displayType = 'samples';
let groupBy = {chips: [], addChip: () => {}, renderChips: () => {}};
function initDropdown(select) {
const wrapper = document.createElement('div');
wrapper.className = 'dropdown';
if (select.classList.contains('f-col')) {
wrapper.classList.add('f-col');
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I’ll locate graphTypeSel (line 422) to decide where to define const tableSelect.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# sed -n '420,438p' scubaduck/
static/index.html
let orderDir = 'ASC';
const orderDirBtn = document.getElementById('order_dir');
const graphTypeSel = document.getElementById('graph_type');
function updateOrderDirButton() {
orderDirBtn.textContent = orderDir + (orderDir === 'ASC' ? ' \u25B2' : ' \u25B
C');
}
function updateDisplayTypeUI() {
const showTable = graphTypeSel.value === 'table';
const showTS = graphTypeSel.value === 'timeseries';
document.getElementById('group_by_field').style.display = showTable || showTS
? 'flex' : 'none';
document.getElementById('aggregate_field').style.display = showTable || showTS
? 'flex' : 'none';
document.getElementById('show_hits_field').style.display = showTable ? 'flex'
: 'none';
document.getElementById('x_axis_field').style.display = showTS ? 'flex' : 'non
e';
document.getElementById('granularity_field').style.display = showTS ? 'flex' :
'none';
document.getElementById('fill_field').style.display = showTS ? 'flex' : 'none'
;
document.querySelectorAll('#column_groups .col-group').forEach(g => {
if (g.querySelector('.col-group-header').textContent.startsWith('Strings'))
{
g.style.display = showTable || showTS ? 'none' : '';
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I'll add const tableSelect = document.getElementById('table'); and update column fetching.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# grep -n "fetch('/api/columns
'" -n scubaduck/static/index.html
462:fetch('/api/columns').then(r => r.json()).then(cols => {
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I’ll review around 460 lines to explore the function.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# sed -n '458,560p' scubaduck/
static/index.html
updateOrderDirButton();
});
updateOrderDirButton();
graphTypeSel.addEventListener('change', updateDisplayTypeUI);
fetch('/api/columns').then(r => r.json()).then(cols => {
const orderSelect = document.getElementById('order_by');
const xAxisSelect = document.getElementById('x_axis');
const groupsEl = document.getElementById('column_groups');
const groups = {
time: {name: 'Time', cols: [], ul: null},
integer: {name: 'Integers', cols: [], ul: null},
string: {name: 'Strings', cols: [], ul: null}
};
cols.forEach(c => {
const t = c.type.toUpperCase();
columnTypes[c.name] = c.type;
allColumns.push(c.name);
baseColumns.push(c.name);
let g = 'string';
if (t.includes('INT')) g = 'integer';
if (t.includes('TIMESTAMP')) {
g = 'time';
timeColumns.push(c.name);
baseTimeColumns.push(c.name);
}
if (g === 'string') {
stringColumns.push(c.name);
baseStringColumns.push(c.name);
} else {
integerColumns.push(c.name);
baseIntegerColumns.push(c.name);
}
groups[g].cols.push(c.name);
if (g !== 'string') {
const o = document.createElement('option');
o.value = c.name;
o.textContent = c.name;
orderSelect.appendChild(o);
}
});
timeColumns.forEach(name => {
const o = document.createElement('option');
o.value = name;
o.textContent = name;
xAxisSelect.appendChild(o);
});
Object.keys(groups).forEach(key => {
const g = groups[key];
const div = document.createElement('div');
div.className = 'col-group';
const header = document.createElement('div');
header.className = 'col-group-header';
header.appendChild(document.createTextNode(g.name + ': '));
const links = document.createElement('span');
links.className = 'links';
const allBtn = document.createElement('a');
allBtn.href = '#';
allBtn.textContent = 'All';
const noneBtn = document.createElement('a');
noneBtn.href = '#';
noneBtn.textContent = 'None';
links.appendChild(allBtn);
links.appendChild(noneBtn);
header.appendChild(links);
div.appendChild(header);
const ul = document.createElement('ul');
g.ul = ul;
g.cols.forEach(name => {
const li = document.createElement('li');
const label = document.createElement('label');
const cb = document.createElement('input');
cb.type = 'checkbox';
cb.value = name;
cb.checked = true;
cb.addEventListener('change', updateSelectedColumns);
label.appendChild(cb);
label.appendChild(document.createTextNode(' ' + name));
li.appendChild(label);
ul.appendChild(li);
});
allBtn.addEventListener('click', e => {
e.preventDefault();
ul.querySelectorAll('input').forEach(cb => (cb.checked = true));
updateSelectedColumns();
});
noneBtn.addEventListener('click', e => {
e.preventDefault();
ul.querySelectorAll('input').forEach(cb => (cb.checked = false));
updateSelectedColumns();
});
div.appendChild(ul);
groupsEl.appendChild(div);
});
document.getElementById('columns_all').addEventListener('click', e => {
e.preventDefault();
groupsEl.querySelectorAll('input').forEach(cb => (cb.checked = true));
updateSelectedColumns();
});
document.getElementById('columns_none').addEventListener('click', e => {
e.preventDefault();
groupsEl.querySelectorAll('input').forEach(cb => (cb.checked = false));
updateSelectedColumns();
});(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# sed -n '560,652p' scubaduck/
static/index.html
});
updateSelectedColumns();
groupBy = document.getElementById('group_by').closest('.field');
initChipInput(groupBy, typed =>
allColumns.filter(c => c.toLowerCase().includes(typed.toLowerCase()))
);
initDropdown(orderSelect);
initDropdown(document.getElementById('aggregate'));
document.getElementById('limit').addEventListener('input', e => {
e.target.dataset.setByUser = '1';
});
updateDisplayTypeUI();
addFilter();
initFromUrl();
});
document.querySelectorAll('#tabs .tab').forEach(btn => {
btn.addEventListener('click', () => {
document.querySelectorAll('#tabs .tab').forEach(t => t.classList.remove('act
ive'));
document.querySelectorAll('.tab-content').forEach(c => c.classList.remove('a
ctive'));
btn.classList.add('active');
document.getElementById(btn.dataset.tab).classList.add('active');
});
});
document.querySelectorAll('.rel-btn').forEach(btn => {
btn.addEventListener('click', () => {
const dd = document.getElementById(btn.dataset.target);
const show = dd.style.display === 'none' || dd.style.display === '';
document.querySelectorAll('.rel-dropdown').forEach(d => (d.style.display = '
none'));
dd.style.display = show ? 'block' : 'none';
});
});
document.querySelectorAll('.rel-dropdown div').forEach(opt => {
opt.addEventListener('click', () => {
const box = opt.closest('.rel-box');
const input = box.querySelector('input');
input.value = opt.dataset.value || opt.textContent;
opt.parentElement.style.display = 'none';
});
});
document.addEventListener('click', e => {
document.querySelectorAll('.rel-dropdown').forEach(dd => {
if (!dd.parentElement.contains(e.target)) dd.style.display = 'none';
});
});
function updateSelectedColumns() {
const base = allColumns.filter(name => {
const cb = document.querySelector(`#column_groups input[value="${name}"]`);
if (!cb || !cb.checked) return false;
if (graphTypeSel.value === 'table' && isStringColumn(name)) return false;
return true;
});
if (graphTypeSel.value === 'table' || graphTypeSel.value === 'timeseries') {
selectedColumns = groupBy.chips.slice();
if (document.getElementById('show_hits').checked) selectedColumns.push('Hits
');
base.forEach(c => {
if (!selectedColumns.includes(c)) selectedColumns.push(c);
});
derivedColumns.forEach(dc => {
if (dc.include && !selectedColumns.includes(dc.name)) selectedColumns.push
(dc.name);
});
} else {
selectedColumns = base.slice();
derivedColumns.forEach(dc => {
if (dc.include) selectedColumns.push(dc.name);
});
}
}
function isStringColumn(name) {
const t = (columnTypes[name] || '').toUpperCase();
return t.includes('CHAR') || t.includes('STRING') || t.includes('VARCHAR');
}
function isIntegerColumn(name) {
const t = (columnTypes[name] || '').toUpperCase();
return t.includes('INT');
}
function isTimeColumn(name) {
const t = (columnTypes[name] || '').toUpperCase();
return t.includes('TIMESTAMP');
}
function formatNumber(val) {
if (typeof val !== 'number') val = Number(val);
if (Number.isNaN(val)) return '';
if (val === 0) return '0';
const abs = Math.abs(val);
if (abs > 999.999) {
const units = [
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I'm updating column loading to fetch data for a specific table and reset arrays accordingly.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# sed -n '458,640p' scubaduck/
static/index.html
updateOrderDirButton();
});
updateOrderDirButton();
graphTypeSel.addEventListener('change', updateDisplayTypeUI);
fetch('/api/columns').then(r => r.json()).then(cols => {
const orderSelect = document.getElementById('order_by');
const xAxisSelect = document.getElementById('x_axis');
const groupsEl = document.getElementById('column_groups');
const groups = {
time: {name: 'Time', cols: [], ul: null},
integer: {name: 'Integers', cols: [], ul: null},
string: {name: 'Strings', cols: [], ul: null}
};
cols.forEach(c => {
const t = c.type.toUpperCase();
columnTypes[c.name] = c.type;
allColumns.push(c.name);
baseColumns.push(c.name);
let g = 'string';
if (t.includes('INT')) g = 'integer';
if (t.includes('TIMESTAMP')) {
g = 'time';
timeColumns.push(c.name);
baseTimeColumns.push(c.name);
}
if (g === 'string') {
stringColumns.push(c.name);
baseStringColumns.push(c.name);
} else {
integerColumns.push(c.name);
baseIntegerColumns.push(c.name);
}
groups[g].cols.push(c.name);
if (g !== 'string') {
const o = document.createElement('option');
o.value = c.name;
o.textContent = c.name;
orderSelect.appendChild(o);
}
});
timeColumns.forEach(name => {
const o = document.createElement('option');
o.value = name;
o.textContent = name;
xAxisSelect.appendChild(o);
});
Object.keys(groups).forEach(key => {
const g = groups[key];
const div = document.createElement('div');
div.className = 'col-group';
const header = document.createElement('div');
header.className = 'col-group-header';
header.appendChild(document.createTextNode(g.name + ': '));
const links = document.createElement('span');
links.className = 'links';
const allBtn = document.createElement('a');
allBtn.href = '#';
allBtn.textContent = 'All';
const noneBtn = document.createElement('a');
noneBtn.href = '#';
noneBtn.textContent = 'None';
links.appendChild(allBtn);
links.appendChild(noneBtn);
header.appendChild(links);
div.appendChild(header);
const ul = document.createElement('ul');
g.ul = ul;
g.cols.forEach(name => {
const li = document.createElement('li');
const label = document.createElement('label');
const cb = document.createElement('input');
cb.type = 'checkbox';
cb.value = name;
cb.checked = true;
cb.addEventListener('change', updateSelectedColumns);
label.appendChild(cb);
label.appendChild(document.createTextNode(' ' + name));
li.appendChild(label);
ul.appendChild(li);
});
allBtn.addEventListener('click', e => {
e.preventDefault();
ul.querySelectorAll('input').forEach(cb => (cb.checked = true));
updateSelectedColumns();
});
noneBtn.addEventListener('click', e => {
e.preventDefault();
ul.querySelectorAll('input').forEach(cb => (cb.checked = false));
updateSelectedColumns();
});
div.appendChild(ul);
groupsEl.appendChild(div);
});
document.getElementById('columns_all').addEventListener('click', e => {
e.preventDefault();
groupsEl.querySelectorAll('input').forEach(cb => (cb.checked = true));
updateSelectedColumns();
});
document.getElementById('columns_none').addEventListener('click', e => {
e.preventDefault();
groupsEl.querySelectorAll('input').forEach(cb => (cb.checked = false));
updateSelectedColumns();
});
updateSelectedColumns();
groupBy = document.getElementById('group_by').closest('.field');
initChipInput(groupBy, typed =>
allColumns.filter(c => c.toLowerCase().includes(typed.toLowerCase()))
);
initDropdown(orderSelect);
initDropdown(document.getElementById('aggregate'));
document.getElementById('limit').addEventListener('input', e => {
e.target.dataset.setByUser = '1';
});
updateDisplayTypeUI();
addFilter();
initFromUrl();
});
document.querySelectorAll('#tabs .tab').forEach(btn => {
btn.addEventListener('click', () => {
document.querySelectorAll('#tabs .tab').forEach(t => t.classList.remove('act
ive'));
document.querySelectorAll('.tab-content').forEach(c => c.classList.remove('a
ctive'));
btn.classList.add('active');
document.getElementById(btn.dataset.tab).classList.add('active');
});
});
document.querySelectorAll('.rel-btn').forEach(btn => {
btn.addEventListener('click', () => {
const dd = document.getElementById(btn.dataset.target);
const show = dd.style.display === 'none' || dd.style.display === '';
document.querySelectorAll('.rel-dropdown').forEach(d => (d.style.display = '
none'));
dd.style.display = show ? 'block' : 'none';
});
});
document.querySelectorAll('.rel-dropdown div').forEach(opt => {
opt.addEventListener('click', () => {
const box = opt.closest('.rel-box');
const input = box.querySelector('input');
input.value = opt.dataset.value || opt.textContent;
opt.parentElement.style.display = 'none';
});
});
document.addEventListener('click', e => {
document.querySelectorAll('.rel-dropdown').forEach(dd => {
if (!dd.parentElement.contains(e.target)) dd.style.display = 'none';
});
});
function updateSelectedColumns() {
const base = allColumns.filter(name => {
const cb = document.querySelector(`#column_groups input[value="${name}"]`);
if (!cb || !cb.checked) return false;
if (graphTypeSel.value === 'table' && isStringColumn(name)) return false;
return true;
});
if (graphTypeSel.value === 'table' || graphTypeSel.value === 'timeseries') {
selectedColumns = groupBy.chips.slice();
if (document.getElementById('show_hits').checked) selectedColumns.push('Hits
');
base.forEach(c => {
if (!selectedColumns.includes(c)) selectedColumns.push(c);
});
derivedColumns.forEach(dc => {
if (dc.include && !selectedColumns.includes(dc.name)) selectedColumns.push
(dc.name);
});
} else {
selectedColumns = base.slice();
derivedColumns.forEach(dc => {
if (dc.include) selectedColumns.push(dc.name);
});
}
}
function isStringColumn(name) {
const t = (columnTypes[name] || '').toUpperCase();
return t.includes('CHAR') || t.includes('STRING') || t.includes('VARCHAR');
}
function isIntegerColumn(name) {
const t = (columnTypes[name] || '').toUpperCase();
return t.includes('INT');
}
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I’ll refactor and move the code into loadColumns(table), then I'll patch it.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# apply_patch<<'PATCH'
> *** Begin Patch
ddEventListener('click', e => {
+ > *** Update File: scubaduck/static/index.html
> @@
> -updateOrderDirButton();
> -graphTypeSel.addEventListener('change', updateDisplayTypeUI);
> -fetch('/api/columns').then(r => r.json()).then(cols => {
> +updateOrderDirButton();
> +graphTypeSel.addEventListener('change', updateDisplayTypeUI);
All('i> +
np> +function loadColumns(table) {
> + return fetch('/api/columns?table=' + encodeURIComponent(table)).then(r => r
.json()).then(cols => {
> + const orderSelect = document.getElementById('order_by');
> + const xAxisSelect = document.getElementById('x_axis');
> + const groupsEl = document.getElementById('column_groups');
> + orderSelect.innerHTML = '';
> + xAxisSelect.innerHTML = '';
> + groupsEl.innerHTML = '';
> + allColumns.length = 0;
> + stringColumns.length = 0;
> + integerColumns.length = 0;
> + timeColumns.length = 0;
> + baseColumns.length = 0;
> + baseStringColumns.length = 0;
> + baseIntegerColumns.length = 0;
> + baseTimeColumns.length = 0;
> + for (const k in columnTypes) delete columnTypes[k];
> + const groups = {
> + time: {name: 'Time', cols: [], ul: null},
> + integer: {name: 'Integers', cols: [], ul: null},
> + string: {name: 'Strings', cols: [], ul: null},
> + };
> + cols.forEach(c => {
> + const t = c.type.toUpperCase();
> + columnTypes[c.name] = c.type;
> + allColumns.push(c.name);
> + baseColumns.push(c.name);
> + let g = 'string';
> + if (t.includes('INT')) g = 'integer';
> + if (t.includes('TIMESTAMP')) {
> + g = 'time';
> + timeColumns.push(c.name);
> + baseTimeColumns.push(c.name);
> + }
> + if (g === 'string') {
> + stringColumns.push(c.name);
> + baseStringColumns.push(c.name);
> + } else {
> + integerColumns.push(c.name);
> + baseIntegerColumns.push(c.name);
> + }
> + groups[g].cols.push(c.name);
> + if (g !== 'string') {
> + const o = document.createElement('option');
> + o.value = c.name;
> + o.textContent = c.name;
> + orderSelect.appendChild(o);
> + }
> + });
> + timeColumns.forEach(name => {
> + const o = document.createElement('option');
> + o.value = name;
> + o.textContent = name;
> + xAxisSelect.appendChild(o);
> + });
> + Object.keys(groups).forEach(key => {
> + const g = groups[key];
> + const div = document.createElement('div');
> + div.className = 'col-group';
> + const header = document.createElement('div');
> + header.className = 'col-group-header';
> + header.appendChild(document.createTextNode(g.name + ': '));
> + const links = document.createElement('span');
> + links.className = 'links';
> + const allBtn = document.createElement('a');
> + allBtn.href = '#';
> + allBtn.textContent = 'All';
> + const noneBtn = document.createElement('a');
> + noneBtn.href = '#';
> + noneBtn.textContent = 'None';
> + links.appendChild(allBtn);
> + links.appendChild(noneBtn);
> + header.appendChild(links);
> + div.appendChild(header);
> + const ul = document.createElement('ul');
> + g.ul = ul;
> + g.cols.forEach(name => {
> + const li = document.createElement('li');
> + const label = document.createElement('label');
> + const cb = document.createElement('input');
> + cb.type = 'checkbox';
> + cb.value = name;
> + cb.checked = true;
> + cb.addEventListener('change', updateSelectedColumns);
> + label.appendChild(cb);
> + label.appendChild(document.createTextNode(' ' + name));
> + li.appendChild(label);
> + ul.appendChild(li);
> + });
> + allBtn.addEventListener('click', e => {
> + e.preventDefault();
> + ul.querySelectorAll('input').forEach(cb => (cb.checked = true));
> + updateSelectedColumns();
> + });
> + noneBtn.addEventListener('click', e => {
> + e.preventDefault();
> + ul.querySelectorAll('input').forEach(cb => (cb.checked = false));
> + updateSelectedColumns();
> + });
> + div.appendChild(ul);
> + groupsEl.appendChild(div);
> + });
> + document.getElementById('columns_all').addEventListener('click', e => {
> + e.preventDefault();
> + groupsEl.querySelectorAll('input').forEach(cb => (cb.checked = true));
> + updateSelectedColumns();
> + });
> + document.getElementById('columns_none').addEventListener('click', e => {
> + e.preventDefault();
> + groupsEl.querySelectorAll('input').forEach(cb => (cb.checked = false));
> + updateSelectedColumns();
> + });
> + updateSelectedColumns();
> + groupBy = document.getElementById('group_by').closest('.field');
> + initChipInput(groupBy, typed =>
> + allColumns.filter(c => c.toLowerCase().includes(typed.toLowerCase()))
> + );
> + initDropdown(orderSelect);
> + initDropdown(document.getElementById('aggregate'));
> + });
> +}
> +
> +let columnsInitialized = false;
> +fetch('/api/tables').then(r => r.json()).then(tables => {
> + tables.forEach(t => {
> + const o = document.createElement('option');
> + o.value = t;
> + o.textContent = t;
> + document.getElementById('table').appendChild(o);
> + });
> + const table = parseSearch().table || tables[0];
> + document.getElementById('table').value = table;
> + loadColumns(table).then(() => {
> + updateDisplayTypeUI();
> + addFilter();
> + initFromUrl();
> + columnsInitialized = true;
> + });
> + document.getElementById('table').addEventListener('change', () => {
> + loadColumns(document.getElementById('table').value).then(() => {
> + if (columnsInitialized) {
> + applyParams(parseSearch());
> + }
> + });
> + });
> +});
> *** End Patch
> PATCH
Success. Updated the following files:
M scubaduck/static/index.html
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#The patch output seems incomplete, so I'll open the file to check it.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# grep -n "loadColumns" -n scu
baduck/static/index.html
463:function loadColumns(table) {
595: loadColumns(table).then(() => {
602: loadColumns(document.getElementById('table').value).then(() => {
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I'll open lines 450-615 to review the new patch changes.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# sed -n '452,615p' scubaduck/
static/index.html
updateSelectedColumns();
}
displayType = graphTypeSel.value;
}
orderDirBtn.addEventListener('click', () => {
orderDir = orderDir === 'ASC' ? 'DESC' : 'ASC';
updateOrderDirButton();
});
updateOrderDirButton();
graphTypeSel.addEventListener('change', updateDisplayTypeUI);
function loadColumns(table) {
return fetch('/api/columns?table=' + encodeURIComponent(table)).then(r => r.js
on()).then(cols => {
const orderSelect = document.getElementById('order_by');
const xAxisSelect = document.getElementById('x_axis');
const groupsEl = document.getElementById('column_groups');
orderSelect.innerHTML = '';
xAxisSelect.innerHTML = '';
groupsEl.innerHTML = '';
allColumns.length = 0;
stringColumns.length = 0;
integerColumns.length = 0;
timeColumns.length = 0;
baseColumns.length = 0;
baseStringColumns.length = 0;
baseIntegerColumns.length = 0;
baseTimeColumns.length = 0;
for (const k in columnTypes) delete columnTypes[k];
const groups = {
time: {name: 'Time', cols: [], ul: null},
integer: {name: 'Integers', cols: [], ul: null},
string: {name: 'Strings', cols: [], ul: null},
};
cols.forEach(c => {
const t = c.type.toUpperCase();
columnTypes[c.name] = c.type;
allColumns.push(c.name);
baseColumns.push(c.name);
let g = 'string';
if (t.includes('INT')) g = 'integer';
if (t.includes('TIMESTAMP')) {
g = 'time';
timeColumns.push(c.name);
baseTimeColumns.push(c.name);
}
if (g === 'string') {
stringColumns.push(c.name);
baseStringColumns.push(c.name);
} else {
integerColumns.push(c.name);
baseIntegerColumns.push(c.name);
}
groups[g].cols.push(c.name);
if (g !== 'string') {
const o = document.createElement('option');
o.value = c.name;
o.textContent = c.name;
orderSelect.appendChild(o);
}
});
timeColumns.forEach(name => {
const o = document.createElement('option');
o.value = name;
o.textContent = name;
xAxisSelect.appendChild(o);
});
Object.keys(groups).forEach(key => {
const g = groups[key];
const div = document.createElement('div');
div.className = 'col-group';
const header = document.createElement('div');
header.className = 'col-group-header';
header.appendChild(document.createTextNode(g.name + ': '));
const links = document.createElement('span');
links.className = 'links';
const allBtn = document.createElement('a');
allBtn.href = '#';
allBtn.textContent = 'All';
const noneBtn = document.createElement('a');
noneBtn.href = '#';
noneBtn.textContent = 'None';
links.appendChild(allBtn);
links.appendChild(noneBtn);
header.appendChild(links);
div.appendChild(header);
const ul = document.createElement('ul');
g.ul = ul;
g.cols.forEach(name => {
const li = document.createElement('li');
const label = document.createElement('label');
const cb = document.createElement('input');
cb.type = 'checkbox';
cb.value = name;
cb.checked = true;
cb.addEventListener('change', updateSelectedColumns);
label.appendChild(cb);
label.appendChild(document.createTextNode(' ' + name));
li.appendChild(label);
ul.appendChild(li);
});
allBtn.addEventListener('click', e => {
e.preventDefault();
ul.querySelectorAll('input').forEach(cb => (cb.checked = true));
updateSelectedColumns();
});
noneBtn.addEventListener('click', e => {
e.preventDefault();
ul.querySelectorAll('input').forEach(cb => (cb.checked = false));
updateSelectedColumns();
});
div.appendChild(ul);
groupsEl.appendChild(div);
});
document.getElementById('columns_all').addEventListener('click', e => {
e.preventDefault();
groupsEl.querySelectorAll('input').forEach(cb => (cb.checked = true));
updateSelectedColumns();
});
document.getElementById('columns_none').addEventListener('click', e => {
e.preventDefault();
groupsEl.querySelectorAll('input').forEach(cb => (cb.checked = false));
updateSelectedColumns();
});
updateSelectedColumns();
groupBy = document.getElementById('group_by').closest('.field');
initChipInput(groupBy, typed =>
allColumns.filter(c => c.toLowerCase().includes(typed.toLowerCase()))
);
initDropdown(orderSelect);
initDropdown(document.getElementById('aggregate'));
});
}
let columnsInitialized = false;
fetch('/api/tables').then(r => r.json()).then(tables => {
tables.forEach(t => {
const o = document.createElement('option');
o.value = t;
o.textContent = t;
document.getElementById('table').appendChild(o);
});
const table = parseSearch().table || tables[0];
document.getElementById('table').value = table;
loadColumns(table).then(() => {
updateDisplayTypeUI();
addFilter();
initFromUrl();
columnsInitialized = true;
});
document.getElementById('table').addEventListener('change', () => {
loadColumns(document.getElementById('table').value).then(() => {
if (columnsInitialized) {
applyParams(parseSearch());
}
});
});
});
const orderSelect = document.getElementById('order_by');
const xAxisSelect = document.getElementById('x_axis');
const groupsEl = document.getElementById('column_groups');
const groups = {
time: {name: 'Time', cols: [], ul: null},
integer: {name: 'Integers', cols: [], ul: null},
string: {name: 'Strings', cols: [], ul: null}
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I noticed old code after the patch—need to remove leftover lines from the fetch call.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# sed -n '612,664p' scubaduck/
static/index.html
const groups = {
time: {name: 'Time', cols: [], ul: null},
integer: {name: 'Integers', cols: [], ul: null},
string: {name: 'Strings', cols: [], ul: null}
};
cols.forEach(c => {
const t = c.type.toUpperCase();
columnTypes[c.name] = c.type;
allColumns.push(c.name);
baseColumns.push(c.name);
let g = 'string';
if (t.includes('INT')) g = 'integer';
if (t.includes('TIMESTAMP')) {
g = 'time';
timeColumns.push(c.name);
baseTimeColumns.push(c.name);
}
if (g === 'string') {
stringColumns.push(c.name);
baseStringColumns.push(c.name);
} else {
integerColumns.push(c.name);
baseIntegerColumns.push(c.name);
}
groups[g].cols.push(c.name);
if (g !== 'string') {
const o = document.createElement('option');
o.value = c.name;
o.textContent = c.name;
orderSelect.appendChild(o);
}
});
timeColumns.forEach(name => {
const o = document.createElement('option');
o.value = name;
o.textContent = name;
xAxisSelect.appendChild(o);
});
Object.keys(groups).forEach(key => {
const g = groups[key];
const div = document.createElement('div');
div.className = 'col-group';
const header = document.createElement('div');
header.className = 'col-group-header';
header.appendChild(document.createTextNode(g.name + ': '));
const links = document.createElement('span');
links.className = 'links';
const allBtn = document.createElement('a');
allBtn.href = '#';
allBtn.textContent = 'All';
const noneBtn = document.createElement('a');
noneBtn.href = '#';
noneBtn.textContent = 'None';
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I'll open lines 602 to 700 to check where the new code ended.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# sed -n '590,710p' scubaduck/
static/index.html
o.textContent = t;
document.getElementById('table').appendChild(o);
});
const table = parseSearch().table || tables[0];
document.getElementById('table').value = table;
loadColumns(table).then(() => {
updateDisplayTypeUI();
addFilter();
initFromUrl();
columnsInitialized = true;
});
document.getElementById('table').addEventListener('change', () => {
loadColumns(document.getElementById('table').value).then(() => {
if (columnsInitialized) {
applyParams(parseSearch());
}
});
});
});
const orderSelect = document.getElementById('order_by');
const xAxisSelect = document.getElementById('x_axis');
const groupsEl = document.getElementById('column_groups');
const groups = {
time: {name: 'Time', cols: [], ul: null},
integer: {name: 'Integers', cols: [], ul: null},
string: {name: 'Strings', cols: [], ul: null}
};
cols.forEach(c => {
const t = c.type.toUpperCase();
columnTypes[c.name] = c.type;
allColumns.push(c.name);
baseColumns.push(c.name);
let g = 'string';
if (t.includes('INT')) g = 'integer';
if (t.includes('TIMESTAMP')) {
g = 'time';
timeColumns.push(c.name);
baseTimeColumns.push(c.name);
}
if (g === 'string') {
stringColumns.push(c.name);
baseStringColumns.push(c.name);
} else {
integerColumns.push(c.name);
baseIntegerColumns.push(c.name);
}
groups[g].cols.push(c.name);
if (g !== 'string') {
const o = document.createElement('option');
o.value = c.name;
o.textContent = c.name;
orderSelect.appendChild(o);
}
});
timeColumns.forEach(name => {
const o = document.createElement('option');
o.value = name;
o.textContent = name;
xAxisSelect.appendChild(o);
});
Object.keys(groups).forEach(key => {
const g = groups[key];
const div = document.createElement('div');
div.className = 'col-group';
const header = document.createElement('div');
header.className = 'col-group-header';
header.appendChild(document.createTextNode(g.name + ': '));
const links = document.createElement('span');
links.className = 'links';
const allBtn = document.createElement('a');
allBtn.href = '#';
allBtn.textContent = 'All';
const noneBtn = document.createElement('a');
noneBtn.href = '#';
noneBtn.textContent = 'None';
links.appendChild(allBtn);
links.appendChild(noneBtn);
header.appendChild(links);
div.appendChild(header);
const ul = document.createElement('ul');
g.ul = ul;
g.cols.forEach(name => {
const li = document.createElement('li');
const label = document.createElement('label');
const cb = document.createElement('input');
cb.type = 'checkbox';
cb.value = name;
cb.checked = true;
cb.addEventListener('change', updateSelectedColumns);
label.appendChild(cb);
label.appendChild(document.createTextNode(' ' + name));
li.appendChild(label);
ul.appendChild(li);
});
allBtn.addEventListener('click', e => {
e.preventDefault();
ul.querySelectorAll('input').forEach(cb => (cb.checked = true));
updateSelectedColumns();
});
noneBtn.addEventListener('click', e => {
e.preventDefault();
ul.querySelectorAll('input').forEach(cb => (cb.checked = false));
updateSelectedColumns();
});
div.appendChild(ul);
groupsEl.appendChild(div);
});
document.getElementById('columns_all').addEventListener('click', e => {
e.preventDefault();
groupsEl.querySelectorAll('input').forEach(cb => (cb.checked = true));
updateSelectedColumns();
});
document.getElementById('columns_none').addEventListener('click', e => {
e.preventDefault();
groupsEl.querySelectorAll('input').forEach(cb => (cb.checked = false));
updateSelectedColumns();
});
updateSelectedColumns();
groupBy = document.getElementById('group_by').closest('.field');
initChipInput(groupBy, typed =>
allColumns.filter(c => c.toLowerCase().includes(typed.toLowerCase()))
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I need to remove code after "const orderSelect..." and find unique string to cut.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# grep -n "initDropdown(orderS
elect)" -n scubaduck/static/index.html
580: initDropdown(orderSelect);
712: initDropdown(orderSelect);
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I'll examine lines around 700 to clean up the leftover patch after the fix.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# sed -n '700,740p' scubaduck/
static/index.html
updateSelectedColumns();
});
document.getElementById('columns_none').addEventListener('click', e => {
e.preventDefault();
groupsEl.querySelectorAll('input').forEach(cb => (cb.checked = false));
updateSelectedColumns();
});
updateSelectedColumns();
groupBy = document.getElementById('group_by').closest('.field');
initChipInput(groupBy, typed =>
allColumns.filter(c => c.toLowerCase().includes(typed.toLowerCase()))
);
initDropdown(orderSelect);
initDropdown(document.getElementById('aggregate'));
document.getElementById('limit').addEventListener('input', e => {
e.target.dataset.setByUser = '1';
});
updateDisplayTypeUI();
addFilter();
initFromUrl();
});
document.querySelectorAll('#tabs .tab').forEach(btn => {
btn.addEventListener('click', () => {
document.querySelectorAll('#tabs .tab').forEach(t => t.classList.remove('act
ive'));
document.querySelectorAll('.tab-content').forEach(c => c.classList.remove('a
ctive'));
btn.classList.add('active');
document.getElementById(btn.dataset.tab).classList.add('active');
});
});
document.querySelectorAll('.rel-btn').forEach(btn => {
btn.addEventListener('click', () => {
const dd = document.getElementById(btn.dataset.target);
const show = dd.style.display === 'none' || dd.style.display === '';
document.querySelectorAll('.rel-dropdown').forEach(d => (d.style.display = '
none'));
dd.style.display = show ? 'block' : 'none';
});
});
document.querySelectorAll('.rel-dropdown div').forEach(opt => {
opt.addEventListener('click', () => {
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I'm still going to scroll for the next part of the content.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# sed -n '740,780p' scubaduck/
static/index.html
opt.addEventListener('click', () => {
const box = opt.closest('.rel-box');
const input = box.querySelector('input');
input.value = opt.dataset.value || opt.textContent;
opt.parentElement.style.display = 'none';
});
});
document.addEventListener('click', e => {
document.querySelectorAll('.rel-dropdown').forEach(dd => {
if (!dd.parentElement.contains(e.target)) dd.style.display = 'none';
});
});
function updateSelectedColumns() {
const base = allColumns.filter(name => {
const cb = document.querySelector(`#column_groups input[value="${name}"]`);
if (!cb || !cb.checked) return false;
if (graphTypeSel.value === 'table' && isStringColumn(name)) return false;
return true;
});
if (graphTypeSel.value === 'table' || graphTypeSel.value === 'timeseries') {
selectedColumns = groupBy.chips.slice();
if (document.getElementById('show_hits').checked) selectedColumns.push('Hits
');
base.forEach(c => {
if (!selectedColumns.includes(c)) selectedColumns.push(c);
});
derivedColumns.forEach(dc => {
if (dc.include && !selectedColumns.includes(dc.name)) selectedColumns.push
(dc.name);
});
} else {
selectedColumns = base.slice();
derivedColumns.forEach(dc => {
if (dc.include) selectedColumns.push(dc.name);
});
}
}
function isStringColumn(name) {
const t = (columnTypes[name] || '').toUpperCase();
return t.includes('CHAR') || t.includes('STRING') || t.includes('VARCHAR');
}
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I'll check earlier lines around 580 to ensure the new function code is correct.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# sed -n '560,620p' scubaduck/
static/index.html
updateSelectedColumns();
});
div.appendChild(ul);
groupsEl.appendChild(div);
});
document.getElementById('columns_all').addEventListener('click', e => {
e.preventDefault();
groupsEl.querySelectorAll('input').forEach(cb => (cb.checked = true));
updateSelectedColumns();
});
document.getElementById('columns_none').addEventListener('click', e => {
e.preventDefault();
groupsEl.querySelectorAll('input').forEach(cb => (cb.checked = false));
updateSelectedColumns();
});
updateSelectedColumns();
groupBy = document.getElementById('group_by').closest('.field');
initChipInput(groupBy, typed =>
allColumns.filter(c => c.toLowerCase().includes(typed.toLowerCase()))
);
initDropdown(orderSelect);
initDropdown(document.getElementById('aggregate'));
});
}
let columnsInitialized = false;
fetch('/api/tables').then(r => r.json()).then(tables => {
tables.forEach(t => {
const o = document.createElement('option');
o.value = t;
o.textContent = t;
document.getElementById('table').appendChild(o);
});
const table = parseSearch().table || tables[0];
document.getElementById('table').value = table;
loadColumns(table).then(() => {
updateDisplayTypeUI();
addFilter();
initFromUrl();
columnsInitialized = true;
});
document.getElementById('table').addEventListener('change', () => {
loadColumns(document.getElementById('table').value).then(() => {
if (columnsInitialized) {
applyParams(parseSearch());
}
});
});
});
const orderSelect = document.getElementById('order_by');
const xAxisSelect = document.getElementById('x_axis');
const groupsEl = document.getElementById('column_groups');
const groups = {
time: {name: 'Time', cols: [], ul: null},
integer: {name: 'Integers', cols: [], ul: null},
string: {name: 'Strings', cols: [], ul: null}
};
cols.forEach(c => {
const t = c.type.toUpperCase();
columnTypes[c.name] = c.type;
allColumns.push(c.name);
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I'll remove the lines starting from const orderSelect up to document.querySelectorAll.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# nl -ba scubaduck/static/inde
x.html | sed -n '620,740p' | head -n 40
620 allColumns.push(c.name);
621 baseColumns.push(c.name);
622 let g = 'string';
623 if (t.includes('INT')) g = 'integer';
624 if (t.includes('TIMESTAMP')) {
625 g = 'time';
626 timeColumns.push(c.name);
627 baseTimeColumns.push(c.name);
628 }
629 if (g === 'string') {
630 stringColumns.push(c.name);
631 baseStringColumns.push(c.name);
632 } else {
633 integerColumns.push(c.name);
634 baseIntegerColumns.push(c.name);
635 }
636 groups[g].cols.push(c.name);
637 if (g !== 'string') {
638 const o = document.createElement('option');
639 o.value = c.name;
640 o.textContent = c.name;
641 orderSelect.appendChild(o);
642 }
643 });
644 timeColumns.forEach(name => {
645 const o = document.createElement('option');
646 o.value = name;
647 o.textContent = name;
648 xAxisSelect.appendChild(o);
649 });
650 Object.keys(groups).forEach(key => {
651 const g = groups[key];
652 const div = document.createElement('div');
653 div.className = 'col-group';
654 const header = document.createElement('div');
655 header.className = 'col-group-header';
656 header.appendChild(document.createTextNode(g.name + ': '));
657 const links = document.createElement('span');
658 links.className = 'links';
659 const allBtn = document.createElement('a');(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# nl -ba scubaduck/static/inde
x.html | sed -n '640,770p' | tail -n 40
731 document.querySelectorAll('.rel-btn').forEach(btn => {
732 btn.addEventListener('click', () => {
733 const dd = document.getElementById(btn.dataset.target);
734 const show = dd.style.display === 'none' || dd.style.display === '';
735 document.querySelectorAll('.rel-dropdown').forEach(d => (d.style.dis
play = 'none'));
736 dd.style.display = show ? 'block' : 'none';
737 });
738 });
739 document.querySelectorAll('.rel-dropdown div').forEach(opt => {
740 opt.addEventListener('click', () => {
741 const box = opt.closest('.rel-box');
742 const input = box.querySelector('input');
743 input.value = opt.dataset.value || opt.textContent;
744 opt.parentElement.style.display = 'none';
745 });
746 });
747 document.addEventListener('click', e => {
748 document.querySelectorAll('.rel-dropdown').forEach(dd => {
749 if (!dd.parentElement.contains(e.target)) dd.style.display = 'none';
750 });
751 });
752
753 function updateSelectedColumns() {
754 const base = allColumns.filter(name => {
755 const cb = document.querySelector(`#column_groups input[value="${nam
e}"]`);
756 if (!cb || !cb.checked) return false;
757 if (graphTypeSel.value === 'table' && isStringColumn(name)) return f
alse;
758 return true;
759 });
760 if (graphTypeSel.value === 'table' || graphTypeSel.value === 'timeseri
es') {
761 selectedColumns = groupBy.chips.slice();
762 if (document.getElementById('show_hits').checked) selectedColumns.pu
sh('Hits');
763 base.forEach(c => {
764 if (!selectedColumns.includes(c)) selectedColumns.push(c);
765 });
766 derivedColumns.forEach(dc => {
767 if (dc.include && !selectedColumns.includes(dc.name)) selectedColu
mns.push(dc.name);
768 });
769 } else {
770 selectedColumns = base.slice();
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I'll verify the remaining lines, starting from around line 609 or 610 for accuracy.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# grep -n "columnsInitialized"
-n scubaduck/static/index.html
585:let columnsInitialized = false;
599: columnsInitialized = true;
603: if (columnsInitialized) {
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I'll check lines 610 to 710 to verify the redundant code for removal.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# nl -ba scubaduck/static/inde
x.html | sed -n '604,640p'
604 applyParams(parseSearch());
605 }
606 });
607 });
608 });
609 const orderSelect = document.getElementById('order_by');
610 const xAxisSelect = document.getElementById('x_axis');
611 const groupsEl = document.getElementById('column_groups');
612 const groups = {
613 time: {name: 'Time', cols: [], ul: null},
614 integer: {name: 'Integers', cols: [], ul: null},
615 string: {name: 'Strings', cols: [], ul: null}
616 };
617 cols.forEach(c => {
618 const t = c.type.toUpperCase();
619 columnTypes[c.name] = c.type;
620 allColumns.push(c.name);
621 baseColumns.push(c.name);
622 let g = 'string';
623 if (t.includes('INT')) g = 'integer';
624 if (t.includes('TIMESTAMP')) {
625 g = 'time';
626 timeColumns.push(c.name);
627 baseTimeColumns.push(c.name);
628 }
629 if (g === 'string') {
630 stringColumns.push(c.name);
631 baseStringColumns.push(c.name);
632 } else {
633 integerColumns.push(c.name);
634 baseIntegerColumns.push(c.name);
635 }
636 groups[g].cols.push(c.name);
637 if (g !== 'string') {
638 const o = document.createElement('option');
639 o.value = c.name;
640 o.textContent = c.name;
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I'm checking lines 640-708 to determine which section to cut.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# nl -ba scubaduck/static/inde
x.html | sed -n '640,708p'
640 o.textContent = c.name;
641 orderSelect.appendChild(o);
642 }
643 });
644 timeColumns.forEach(name => {
645 const o = document.createElement('option');
646 o.value = name;
647 o.textContent = name;
648 xAxisSelect.appendChild(o);
649 });
650 Object.keys(groups).forEach(key => {
651 const g = groups[key];
652 const div = document.createElement('div');
653 div.className = 'col-group';
654 const header = document.createElement('div');
655 header.className = 'col-group-header';
656 header.appendChild(document.createTextNode(g.name + ': '));
657 const links = document.createElement('span');
658 links.className = 'links';
659 const allBtn = document.createElement('a');
660 allBtn.href = '#';
661 allBtn.textContent = 'All';
662 const noneBtn = document.createElement('a');
663 noneBtn.href = '#';
664 noneBtn.textContent = 'None';
665 links.appendChild(allBtn);
666 links.appendChild(noneBtn);
667 header.appendChild(links);
668 div.appendChild(header);
669 const ul = document.createElement('ul');
670 g.ul = ul;
671 g.cols.forEach(name => {
672 const li = document.createElement('li');
673 const label = document.createElement('label');
674 const cb = document.createElement('input');
675 cb.type = 'checkbox';
676 cb.value = name;
677 cb.checked = true;
678 cb.addEventListener('change', updateSelectedColumns);
679 label.appendChild(cb);
680 label.appendChild(document.createTextNode(' ' + name));
681 li.appendChild(label);
682 ul.appendChild(li);
683 });
684 allBtn.addEventListener('click', e => {
685 e.preventDefault();
686 ul.querySelectorAll('input').forEach(cb => (cb.checked = true));
687 updateSelectedColumns();
688 });
689 noneBtn.addEventListener('click', e => {
690 e.preventDefault();
691 ul.querySelectorAll('input').forEach(cb => (cb.checked = false));
692 updateSelectedColumns();
693 });
694 div.appendChild(ul);
695 groupsEl.appendChild(div);
696 });
697 document.getElementById('columns_all').addEventListener('click', e =>
{
698 e.preventDefault();
699 groupsEl.querySelectorAll('input').forEach(cb => (cb.checked = true)
);
700 updateSelectedColumns();
701 });
702 document.getElementById('columns_none').addEventListener('click', e =>
{
703 e.preventDefault();
704 groupsEl.querySelectorAll('input').forEach(cb => (cb.checked = false
));
705 updateSelectedColumns();
706 });
707 updateSelectedColumns();
708 groupBy = document.getElementById('group_by').closest('.field');
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I'll check lines starting from 708 onward for relevant content.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# nl -ba scubaduck/static/inde
x.html | sed -n '708,742p'
708 groupBy = document.getElementById('group_by').closest('.field');
709 initChipInput(groupBy, typed =>
710 allColumns.filter(c => c.toLowerCase().includes(typed.toLowerCase())
)
711 );
712 initDropdown(orderSelect);
713 initDropdown(document.getElementById('aggregate'));
714 document.getElementById('limit').addEventListener('input', e => {
715 e.target.dataset.setByUser = '1';
716 });
717 updateDisplayTypeUI();
718 addFilter();
719 initFromUrl();
720 });
721
722 document.querySelectorAll('#tabs .tab').forEach(btn => {
723 btn.addEventListener('click', () => {
724 document.querySelectorAll('#tabs .tab').forEach(t => t.classList.rem
ove('active'));
725 document.querySelectorAll('.tab-content').forEach(c => c.classList.r
emove('active'));
726 btn.classList.add('active');
727 document.getElementById(btn.dataset.tab).classList.add('active');
728 });
729 });
730
731 document.querySelectorAll('.rel-btn').forEach(btn => {
732 btn.addEventListener('click', () => {
733 const dd = document.getElementById(btn.dataset.target);
734 const show = dd.style.display === 'none' || dd.style.display === '';
735 document.querySelectorAll('.rel-dropdown').forEach(d => (d.style.dis
play = 'none'));
736 dd.style.display = show ? 'block' : 'none';
737 });
738 });
739 document.querySelectorAll('.rel-dropdown div').forEach(opt => {
740 opt.addEventListener('click', () => {
741 const box = opt.closest('.rel-box');
742 const input = box.querySelector('input');
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I'll remove the leftover fetch call and lines 609-720. Time to patch it!
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# apply_patch<<'PATCH'
mit').addEventListene> *** Begin Patch
> *** Update File: scubaduck/static/index.html
> @@
);
> - document.getElementById('table').addEventListener('change', () => {
> - loadColumns(document.getElementById('table').value).then(() => {
> - if (columnsInitialized) {
> - applyParams(parseSearch());
> - }
> - });
> - });
> -});
> - const orderSelect = document.getElementById('order_by');
);
> - const xAxisSelect = document.getElementById('x_axis');
> - const groupsEl = document.getElementById('column_groups');
> - const groups = {
> - time: {name: 'Time', cols: [], ul: null},
> - integer: {name: 'Integers', cols: [], ul: null},
> - string: {name: 'Strings', cols: [], ul: null}
> - };
> - cols.forEach(c => {
> - const t = c.type.toUpperCase();
> - columnTypes[c.name] = c.type;
> - allColumns.push(c.name);
> - baseColumns.push(c.name);
> - let g = 'string';
> - if (t.includes('INT')) g = 'integer';
> - if (t.includes('TIMESTAMP')) {
> - g = 'time';
> - timeColumns.push(c.name);
> - baseTimeColumns.push(c.name);
> - }
> - if (g === 'string') {
> - stringColumns.push(c.name);
> - baseStringColumns.push(c.name);
> - } else {
> - integerColumns.push(c.name);
> - baseIntegerColumns.push(c.name);
> - }
> - groups[g].cols.push(c.name);
> - if (g !== 'string') {
> - const o = document.createElement('option');
> - o.value = c.name;
> - o.textContent = c.name;
> - orderSelect.appendChild(o);
> - }
> - });
> - timeColumns.forEach(name => {
> - const o = document.createElement('option');
> - o.value = name;
> - o.textContent = name;
> - xAxisSelect.appendChild(o);
> - });
> - Object.keys(groups).forEach(key => {
> - const g = groups[key];
> - const div = document.createElement('div');
> - div.className = 'col-group';
> - const header = document.createElement('div');
> - header.className = 'col-group-header';
> - header.appendChild(document.createTextNode(g.name + ': '));
> - const links = document.createElement('span');
> - links.className = 'links';
> - const allBtn = document.createElement('a');
> - allBtn.href = '#';
> - allBtn.textContent = 'All';
> - const noneBtn = document.createElement('a');
> - noneBtn.href = '#';
> - noneBtn.textContent = 'None';
> - links.appendChild(allBtn);
> - links.appendChild(noneBtn);
> - header.appendChild(links);
> - div.appendChild(header);
> - const ul = document.createElement('ul');
> - g.ul = ul;
> - g.cols.forEach(name => {
> - const li = document.createElement('li');
> - const label = document.createElement('label');
> - const cb = document.createElement('input');
> - cb.type = 'checkbox';
> - cb.value = name;
> - cb.checked = true;
> - cb.addEventListener('change', updateSelectedColumns);
> - label.appendChild(cb);
> - label.appendChild(document.createTextNode(' ' + name));
> - li.appendChild(label);
> - ul.appendChild(li);
> - });
> - allBtn.addEventListener('click', e => {
> - e.preventDefault();
> - ul.querySelectorAll('input').forEach(cb => (cb.checked = true));
> - updateSelectedColumns();
> - });
> - noneBtn.addEventListener('click', e => {
> - e.preventDefault();
> - ul.querySelectorAll('input').forEach(cb => (cb.checked = false));
> - updateSelectedColumns();
> - });
> - div.appendChild(ul);
> - groupsEl.appendChild(div);
> - });
> - document.getElementById('columns_all').addEventListener('click', e => {
> - e.preventDefault();
> - groupsEl.querySelectorAll('input').forEach(cb => (cb.checked = true));
> - updateSelectedColumns();
> - });
> - document.getElementById('columns_none').addEventListener('click', e => {
> - e.preventDefault();
> - groupsEl.querySelectorAll('input').forEach(cb => (cb.checked = false));
> - updateSelectedColumns();
> - });
> - updateSelectedColumns();
> - groupBy = document.getElementById('group_by').closest('.field');
> - initChipInput(groupBy, typed =>
> - allColumns.filter(c => c.toLowerCase().includes(typed.toLowerCase()))
> - );
> - initDropdown(orderSelect);
> - initDropdown(document.getElementById('aggregate'));
> - document.getElementById('limit').addEventListener('input', e => {
> - e.target.dataset.setByUser = '1';
> - });
> - updateDisplayTypeUI();
> - addFilter();
> - initFromUrl();
> -});
> + document.getElementById('table').addEventListener('change', () => {
> + loadColumns(document.getElementById('table').value).then(() => {
> + if (columnsInitialized) {
> + applyParams(parseSearch());
> + }
> + });
> + });
> +});
> *** End Patch
> PATCH
Success. Updated the following files:
M scubaduck/static/index.html
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I’ll double-check to ensure I didn’t accidentally remove too much from the patch.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# grep -n "document.querySelec
torAll('#tabs" -n scubaduck/static/index.html
610:document.querySelectorAll('#tabs .tab').forEach(btn => {
612: document.querySelectorAll('#tabs .tab').forEach(t => t.classList.remove(
'active'));
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I'll open lines between 600 and 630 to verify the patch.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# sed -n '600,636p' scubaduck/
static/index.html
});
document.getElementById('table').addEventListener('change', () => {
loadColumns(document.getElementById('table').value).then(() => {
if (columnsInitialized) {
applyParams(parseSearch());
}
});
});
});
document.querySelectorAll('#tabs .tab').forEach(btn => {
btn.addEventListener('click', () => {
document.querySelectorAll('#tabs .tab').forEach(t => t.classList.remove('act
ive'));
document.querySelectorAll('.tab-content').forEach(c => c.classList.remove('a
ctive'));
btn.classList.add('active');
document.getElementById(btn.dataset.tab).classList.add('active');
});
});
document.querySelectorAll('.rel-btn').forEach(btn => {
btn.addEventListener('click', () => {
const dd = document.getElementById(btn.dataset.target);
const show = dd.style.display === 'none' || dd.style.display === '';
document.querySelectorAll('.rel-dropdown').forEach(d => (d.style.display = '
none'));
dd.style.display = show ? 'block' : 'none';
});
});
document.querySelectorAll('.rel-dropdown div').forEach(opt => {
opt.addEventListener('click', () => {
const box = opt.closest('.rel-box');
const input = box.querySelector('input');
input.value = opt.dataset.value || opt.textContent;
opt.parentElement.style.display = 'none';
});
});
document.addEventListener('click', e => {
document.querySelectorAll('.rel-dropdown').forEach(dd => {
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I'll look into updating functions like parseSearch, paramsToSearch, and others to handle table parameters.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# grep -n "parseSearch" -n scu
baduck/static/index.html
593: const table = parseSearch().table || tables[0];
604: applyParams(parseSearch());
999:function parseSearch() {
1025: const params = parseSearch();
1032: const params = e.state || parseSearch();
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I'll check the lines that contain the parseSearch function.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# sed -n '960,1040p' scubaduck
/static/index.html
if (params.group_by) {
groupBy.chips.splice(0, groupBy.chips.length, ...params.group_by);
groupBy.renderChips();
}
if (params.aggregate) document.getElementById('aggregate').value = params.aggr
egate;
document.getElementById('show_hits').checked = params.show_hits ?? true;
document.querySelectorAll('#column_groups input').forEach(cb => {
cb.checked = !params.columns || params.columns.includes(cb.value);
});
updateSelectedColumns();
const dlist = document.getElementById('derived_list');
dlist.innerHTML = '';
derivedColumns.splice(0, derivedColumns.length);
if (params.derived_columns && params.derived_columns.length) {
params.derived_columns.forEach(dc => addDerived(dc));
}
refreshDerivedColumns();
const list = document.getElementById('filter_list');
list.innerHTML = '';
if (params.filters && params.filters.length) {
params.filters.forEach(f => {
addFilter();
const el = list.lastElementChild;
const colSel = el.querySelector('.f-col select') || el.querySelector('.f-c
ol');
colSel.value = f.column;
colSel.dispatchEvent(new Event('change'));
el.querySelector('.f-op').value = f.op;
el.querySelector('.f-op').dispatchEvent(new Event('change'));
if (f.value !== null && f.op !== 'empty' && f.op !== '!empty') {
const values = Array.isArray(f.value) ? f.value : [f.value];
values.forEach(v => el.addChip(v));
el.renderChips();
}
});
} else {
addFilter();
}
}
function parseSearch() {
const sp = new URLSearchParams(window.location.search);
const params = {};
if (sp.has('start')) params.start = sp.get('start');
if (sp.has('end')) params.end = sp.get('end');
if (sp.has('order_by')) params.order_by = sp.get('order_by');
if (sp.has('order_dir')) params.order_dir = sp.get('order_dir');
if (sp.has('limit')) params.limit = parseInt(sp.get('limit'), 10);
if (sp.has('columns')) params.columns = sp.get('columns').split(',').filter(c
=> c);
if (sp.has('filters')) {
try { params.filters = JSON.parse(sp.get('filters')); } catch(e) { params.fi
lters = []; }
}
if (sp.has('graph_type')) params.graph_type = sp.get('graph_type');
if (sp.has('group_by')) params.group_by = sp.get('group_by').split(',').filter
(c => c);
if (sp.has('aggregate')) params.aggregate = sp.get('aggregate');
if (sp.has('show_hits')) params.show_hits = sp.get('show_hits') === '1';
if (sp.has('x_axis')) params.x_axis = sp.get('x_axis');
if (sp.has('granularity')) params.granularity = sp.get('granularity');
if (sp.has('fill')) params.fill = sp.get('fill');
if (sp.has('derived_columns')) {
try { params.derived_columns = JSON.parse(sp.get('derived_columns')); } catc
h(e) { params.derived_columns = []; }
}
return params;
}
function initFromUrl() {
const params = parseSearch();
history.replaceState(params, '', paramsToSearch(params));
applyParams(params);
dive(false);
}
window.addEventListener('popstate', e => {
const params = e.state || parseSearch();
applyParams(params);
dive(false);
});
let originalRows = [];
let sortState = {index: null, dir: null};
function renderTable(rows) {
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I'll add the table parameter in parseSearch, paramsToSearch, applyParams, and collectParams. Let's check paramsToSearch.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# grep -n "paramsToSearch" -n
scubaduck/static/index.html
850: history.pushState(params, '', paramsToSearch(params));
919:function paramsToSearch(params) {
1026: history.replaceState(params, '', paramsToSearch(params));
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I'll focus on the code near line 919 for now.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# sed -n '912,965p' scubaduck/
static/index.html
payload.x_axis = document.getElementById('x_axis').value;
payload.granularity = document.getElementById('granularity').value;
payload.fill = document.getElementById('fill').value;
}
return payload;
}
function paramsToSearch(params) {
const sp = new URLSearchParams();
if (params.start) sp.set('start', params.start);
if (params.end) sp.set('end', params.end);
if (params.order_by) sp.set('order_by', params.order_by);
if (params.order_dir) sp.set('order_dir', params.order_dir);
if (params.limit !== null && params.limit !== undefined) sp.set('limit', param
s.limit);
if (params.columns && params.columns.length) sp.set('columns', params.columns.
join(','));
if (params.filters && params.filters.length) sp.set('filters', JSON.stringify(
params.filters));
if (params.derived_columns && params.derived_columns.length) sp.set('derived_c
olumns', JSON.stringify(params.derived_columns));
if (params.graph_type) sp.set('graph_type', params.graph_type);
if (params.graph_type === 'table' || params.graph_type === 'timeseries') {
if (params.group_by && params.group_by.length) sp.set('group_by', params.gro
up_by.join(','));
if (params.aggregate) sp.set('aggregate', params.aggregate);
if (params.show_hits) sp.set('show_hits', '1');
}
if (params.graph_type === 'timeseries') {
if (params.x_axis) sp.set('x_axis', params.x_axis);
if (params.granularity) sp.set('granularity', params.granularity);
if (params.fill) sp.set('fill', params.fill);
}
const qs = sp.toString();
return qs ? '?' + qs : '';
}
function applyParams(params) {
document.getElementById('start').value = params.start || '';
document.getElementById('end').value = params.end || '';
if (params.order_by) {
document.getElementById('order_by').value = params.order_by;
}
orderDir = params.order_dir || 'ASC';
updateOrderDirButton();
if (params.limit !== undefined && params.limit !== null) {
document.getElementById('limit').value = params.limit;
}
graphTypeSel.value = params.graph_type || 'samples';
updateDisplayTypeUI();
if (params.x_axis) document.getElementById('x_axis').value = params.x_axis;
if (params.granularity) document.getElementById('granularity').value = params.
granularity;
if (params.fill) document.getElementById('fill').value = params.fill;
if (params.group_by) {
groupBy.chips.splice(0, groupBy.chips.length, ...params.group_by);
groupBy.renderChips();
}
if (params.aggregate) document.getElementById('aggregate').value = params.aggr
egate;
document.getElementById('show_hits').checked = params.show_hits ?? true;
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I'll include a table parameter and search for the "collectParams" function around line 830.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# grep -n "collectParams" -n s
cubaduck/static/index.html
848: const params = collectParams();
877:function collectParams() {
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#Let me take a look at about 870 lines.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# sed -n '870,912p' scubaduck/
static/index.html
showResults(data);
})
.catch(err => {
showError(err);
});
}
function collectParams() {
updateSelectedColumns();
const payload = {
start: document.getElementById('start').value,
end: document.getElementById('end').value,
order_by: document.getElementById('order_by').value,
order_dir: orderDir,
limit: parseInt(document.getElementById('limit').value, 10),
columns: selectedColumns.filter(c =>
c !== 'Hits' && !derivedColumns.some(dc => dc.name === c)
),
graph_type: graphTypeSel.value,
filters: Array.from(document.querySelectorAll('#filters .filter')).map(f =>
{
const chips = f.chips || [];
const op = f.querySelector('.f-op').value;
let value = null;
if (op !== 'empty' && op !== '!empty') {
value = chips.length === 0 ? null : (chips.length === 1 ? chips[0] : chi
ps);
}
const colSel = f.querySelector('.f-col select') || f.querySelector('.f-col
');
return {column: colSel.value, op, value};
}),
derived_columns: Array.from(document.querySelectorAll('#derived_list .derive
d')).map(d => ({
type: d.querySelector('.d-type').value,
name: d.querySelector('.d-name').value,
expr: d.querySelector('.d-expr').value,
include: d.querySelector('.d-use').checked,
}))
};
if (graphTypeSel.value === 'table' || graphTypeSel.value === 'timeseries') {
payload.group_by = groupBy.chips || [];
payload.aggregate = document.getElementById('aggregate').value;
payload.show_hits = document.getElementById('show_hits').checked;
}
if (graphTypeSel.value === 'timeseries') {
payload.x_axis = document.getElementById('x_axis').value;(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# sed -n '912,944p' scubaduck/
static/index.html
payload.x_axis = document.getElementById('x_axis').value;
payload.granularity = document.getElementById('granularity').value;
payload.fill = document.getElementById('fill').value;
}
return payload;
}
function paramsToSearch(params) {
const sp = new URLSearchParams();
if (params.start) sp.set('start', params.start);
if (params.end) sp.set('end', params.end);
if (params.order_by) sp.set('order_by', params.order_by);
if (params.order_dir) sp.set('order_dir', params.order_dir);
if (params.limit !== null && params.limit !== undefined) sp.set('limit', param
s.limit);
if (params.columns && params.columns.length) sp.set('columns', params.columns.
join(','));
if (params.filters && params.filters.length) sp.set('filters', JSON.stringify(
params.filters));
if (params.derived_columns && params.derived_columns.length) sp.set('derived_c
olumns', JSON.stringify(params.derived_columns));
if (params.graph_type) sp.set('graph_type', params.graph_type);
if (params.graph_type === 'table' || params.graph_type === 'timeseries') {
if (params.group_by && params.group_by.length) sp.set('group_by', params.gro
up_by.join(','));
if (params.aggregate) sp.set('aggregate', params.aggregate);
if (params.show_hits) sp.set('show_hits', '1');
}
if (params.graph_type === 'timeseries') {
if (params.x_axis) sp.set('x_axis', params.x_axis);
if (params.granularity) sp.set('granularity', params.granularity);
if (params.fill) sp.set('fill', params.fill);
}
const qs = sp.toString();
return qs ? '?' + qs : '';
}
function applyParams(params) {
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I need to update the table parameter across different functions and patch it.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# apply_patch<<'PATCH'
> *** Begin Patch
> *** Update File: scubaduck/static/index.html
> @@
> const payload = {
> + table: document.getElementById('table').value,
> start: document.getElementById('start').value,
> *** End Patch
> PATCH
Success. Updated the following files:
M scubaduck/static/index.html
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I’m updating the paramsToSearch to address the issue.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# apply_patch<<'PATCH'
> *** Begin Patch
> *** Update File: scubaduck/static/index.html
> @@
> const sp = new URLSearchParams();
> + if (params.table) sp.set('table', params.table);
> *** End Patch
> PATCH
Success. Updated the following files:
M scubaduck/static/index.html
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I'm going to take a look at the parseSearch function next.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# apply_patch<<'PATCH'
> *** Begin Patch
> *** Update File: scubaduck/static/index.html
> @@
> const params = {};
> + if (sp.has('table')) params.table = sp.get('table');
> *** End Patch
> PATCH
Success. Updated the following files:
M scubaduck/static/index.html
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I’ll search to set the table selection value correctly.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# grep -n "applyParams" -n scu
baduck/static/index.html
604: applyParams(parseSearch());
946:function applyParams(params) {
1030: applyParams(params);
1036: applyParams(params);
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I’m unfolding about 946 lines to explore.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# sed -n '946,1004p' scubaduck
/static/index.html
function applyParams(params) {
document.getElementById('start').value = params.start || '';
document.getElementById('end').value = params.end || '';
if (params.order_by) {
document.getElementById('order_by').value = params.order_by;
}
orderDir = params.order_dir || 'ASC';
updateOrderDirButton();
if (params.limit !== undefined && params.limit !== null) {
document.getElementById('limit').value = params.limit;
}
graphTypeSel.value = params.graph_type || 'samples';
updateDisplayTypeUI();
if (params.x_axis) document.getElementById('x_axis').value = params.x_axis;
if (params.granularity) document.getElementById('granularity').value = params.
granularity;
if (params.fill) document.getElementById('fill').value = params.fill;
if (params.group_by) {
groupBy.chips.splice(0, groupBy.chips.length, ...params.group_by);
groupBy.renderChips();
}
if (params.aggregate) document.getElementById('aggregate').value = params.aggr
egate;
document.getElementById('show_hits').checked = params.show_hits ?? true;
document.querySelectorAll('#column_groups input').forEach(cb => {
cb.checked = !params.columns || params.columns.includes(cb.value);
});
updateSelectedColumns();
const dlist = document.getElementById('derived_list');
dlist.innerHTML = '';
derivedColumns.splice(0, derivedColumns.length);
if (params.derived_columns && params.derived_columns.length) {
params.derived_columns.forEach(dc => addDerived(dc));
}
refreshDerivedColumns();
const list = document.getElementById('filter_list');
list.innerHTML = '';
if (params.filters && params.filters.length) {
params.filters.forEach(f => {
addFilter();
const el = list.lastElementChild;
const colSel = el.querySelector('.f-col select') || el.querySelector('.f-c
ol');
colSel.value = f.column;
colSel.dispatchEvent(new Event('change'));
el.querySelector('.f-op').value = f.op;
el.querySelector('.f-op').dispatchEvent(new Event('change'));
if (f.value !== null && f.op !== 'empty' && f.op !== '!empty') {
const values = Array.isArray(f.value) ? f.value : [f.value];
values.forEach(v => el.addChip(v));
el.renderChips();
}
});
} else {
addFilter();
}
}
function parseSearch() {
const sp = new URLSearchParams(window.location.search);
const params = {};
if (sp.has('table')) params.table = sp.get('table');
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I’ll place the setting table value at the top for better structure.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# apply_patch<<'PATCH'
> *** Begin Patch
> *** Update File: scubaduck/static/index.html
> @@
> function applyParams(params) {
> + if (params.table) document.getElementById('table').value = params.table;
> document.getElementById('start').value = params.start || '';
> *** End Patch
> PATCH
Success. Updated the following files:
M scubaduck/static/index.html
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I'll update parseSearch to include table when building search from history. Opening around line 992.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# sed -n '996,1036p' scubaduck
/static/index.html
});
} else {
addFilter();
}
}
function parseSearch() {
const sp = new URLSearchParams(window.location.search);
const params = {};
if (sp.has('table')) params.table = sp.get('table');
if (sp.has('start')) params.start = sp.get('start');
if (sp.has('end')) params.end = sp.get('end');
if (sp.has('order_by')) params.order_by = sp.get('order_by');
if (sp.has('order_dir')) params.order_dir = sp.get('order_dir');
if (sp.has('limit')) params.limit = parseInt(sp.get('limit'), 10);
if (sp.has('columns')) params.columns = sp.get('columns').split(',').filter(c
=> c);
if (sp.has('filters')) {
try { params.filters = JSON.parse(sp.get('filters')); } catch(e) { params.fi
lters = []; }
}
if (sp.has('graph_type')) params.graph_type = sp.get('graph_type');
if (sp.has('group_by')) params.group_by = sp.get('group_by').split(',').filter
(c => c);
if (sp.has('aggregate')) params.aggregate = sp.get('aggregate');
if (sp.has('show_hits')) params.show_hits = sp.get('show_hits') === '1';
if (sp.has('x_axis')) params.x_axis = sp.get('x_axis');
if (sp.has('granularity')) params.granularity = sp.get('granularity');
if (sp.has('fill')) params.fill = sp.get('fill');
if (sp.has('derived_columns')) {
try { params.derived_columns = JSON.parse(sp.get('derived_columns')); } catc
h(e) { params.derived_columns = []; }
}
return params;
}
function initFromUrl() {
const params = parseSearch();
history.replaceState(params, '', paramsToSearch(params));
applyParams(params);
dive(false);
}
window.addEventListener('popstate', e => {
const params = e.state || parseSearch();
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I'm updating table parsing, patches, and the parseSearch function. Now it's time for tests.
import duckdb
from scubaduck import server
import pytest
def test_basic_query() -> None:
app = server.app
client = app.test_client()
payload = {
"start": "2024-01-01 00:00:00",
"end": "2024-01-02 00:00:00",
"order_by": "timestamp",
"order_dir": "ASC",
"limit": 10,
"columns": ["timestamp", "event", "value", "user"],
"filters": [],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert data
rows = data["rows"]
# We expect first three rows (until 2024-01-02 00:00:00)
assert len(rows) == 3
assert rows[0][1] == "login"
assert rows[1][1] == "logout"
def test_js_served() -> None:
app = server.app
client = app.test_client()
rv = client.get("/js/chip_input.js")
assert rv.status_code == 200
assert b"initChipInput" in rv.data
def test_filter_multi_token() -> None:
app = server.app
client = app.test_client()
payload = {
"start": "2024-01-01 00:00:00",
"end": "2024-01-02 03:00:00",
"order_by": "timestamp",
"limit": 10,
"columns": ["timestamp", "event", "value", "user"],
"filters": [{"column": "user", "op": "=", "value": ["alice", "charlie"]}
],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert data
rows = data["rows"]
# Should only return rows for alice and charlie
assert len(rows) == 3
assert rows[0][3] == "alice"
assert rows[-1][3] == "charlie"
def test_empty_filter_is_noop() -> None:
app = server.app
client = app.test_client()
base_payload = {
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"limit": 100,
"columns": ["timestamp", "event", "value", "user"],
}
no_filter = {**base_payload, "filters": []}
empty_filter = {
**base_payload,
"filters": [{"column": "user", "op": "=", "value": None}],
}
rv1 = client.post(
"/api/query", data=json.dumps(no_filter), content_type="application/json
"
)
rv2 = client.post(
"/api/query", data=json.dumps(empty_filter), content_type="application/j
son"
)
rows1 = rv1.get_json()["rows"]
rows2 = rv2.get_json()["rows"]
assert rows1 == rows2
def test_select_columns() -> None:
app = server.app
client = app.test_client()
payload = {
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"order_by": "timestamp",
"limit": 10,
"columns": ["timestamp", "user"],
"filters": [],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert data
rows = data["rows"]
assert len(rows[0]) == 2
assert rows[0][1] == "alice"
def test_string_filter_ops() -> None:
app = server.app
client = app.test_client()
base = {
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"order_by": "timestamp",
"limit": 100,
"columns": ["timestamp", "event", "value", "user"],
}
contains = {
**base,
"filters": [{"column": "user", "op": "contains", "value": "ali"}],
}
rv = client.post(
"/api/query", data=json.dumps(contains), content_type="application/json"
)
rows = rv.get_json()["rows"]
assert all("ali" in r[3] for r in rows)
regex = {
**base,
"filters": [{"column": "user", "op": "~", "value": "^a.*"}],
}
rv = client.post(
"/api/query", data=json.dumps(regex), content_type="application/json"
)
rows = rv.get_json()["rows"]
assert all(r[3].startswith("a") for r in rows)
assert len(rows) == 2
not_empty = {**base, "filters": [{"column": "user", "op": "!empty"}]}
rv = client.post(
"/api/query", data=json.dumps(not_empty), content_type="application/json
"
)
assert len(rv.get_json()["rows"]) == 4
def _make_payload() -> dict[str, object]:
return {
"start": "2024-01-01 00:00:00",
"end": "2024-01-02 00:00:00",
"order_by": "timestamp",
"order_dir": "ASC",
"limit": 10,
"columns": ["timestamp", "event", "value", "user"],
"filters": [],
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I'll add the table parameter at _make_payload.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# grep -n "_make_payload" -n t
ests/test_server.py
152:def _make_payload() -> dict[str, object]:
195: payload = _make_payload()
209: payload = _make_payload()_server.py
def _make_payload() -> dict[str, object]:
return {
"start": "2024-01-01 00:00:00",
"end": "2024-01-02 00:00:00",
"order_by": "timestamp",
"order_dir": "ASC",
"limit": 10,
"columns": ["timestamp", "event", "value", "user"],
"filters": [],
}
def test_database_types(tmp_path: Path) -> None:
csv_file = tmp_path / "events.csv"
csv_file.write_text(Path("scubaduck/sample.csv").read_text())
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I'll add 'table': 'events' and update tests to check for default events and table specification.
> *** Begin Patch
> *** Update File: tests/test_server.py
> @@
> def _make_payload() -> dict[str, object]:
> return {
> + "table": "events",
> "start": "2024-01-01 00:00:00",
> "end": "2024-01-02 00:00:00",
> "order_by": "timestamp",
> *** End Patch
> PATCH
Success. Updated the following files:
M tests/test_server.py
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I'll check the last test and append the new server test at the bottom.
.py
assert rv.status_code == 200
rows = data["rows"]
assert all(r[2] == r[1] * 2 for r in rows)
def test_default_start_end_returned() -> None:
app = server.app
client = app.test_client()
payload = {
"order_by": "timestamp",
"limit": 5,
"columns": ["timestamp"],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert data["start"] == "2024-01-01 00:00:00"
assert data["end"] == "2024-01-02 03:00:00"
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I'm thinking of appending the new test either after 'test_database_types' or at the bottom.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# grep -n "test_database_types
" -n tests/test_server.py
165:def test_database_types(tmp_path: Path) -> None:
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I'll check around line 220 to decide where to add the new test.
}
def test_database_types(tmp_path: Path) -> None:
csv_file = tmp_path / "events.csv"
csv_file.write_text(Path("scubaduck/sample.csv").read_text())
sqlite_file = tmp_path / "events.sqlite"
import sqlite3
conn = sqlite3.connect(sqlite_file)
conn.execute(
"CREATE TABLE events (timestamp TEXT, event TEXT, value INTEGER, user TE
XT)"
)
with open(csv_file) as f:
next(f)
for line in f:
ts, ev, val, user = line.strip().split(",")
conn.execute(
"INSERT INTO events VALUES (?, ?, ?, ?)", (ts, ev, int(val), use
r)
)
conn.commit()
conn.close() # pyright: ignore[reportUnknownMemberType, reportAttributeAcce
ssIssue]
duckdb_file = tmp_path / "events.duckdb"
con = duckdb.connect(duckdb_file)
con.execute(
f"CREATE TABLE events AS SELECT * FROM read_csv_auto('{csv_file.as_posix
()}')"
)
con.close() # pyright: ignore[reportUnknownMemberType, reportAttributeAcces
sIssue]
for db in (csv_file, sqlite_file, duckdb_file):
app = server.create_app(db)
client = app.test_client()
payload = _make_payload()
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/js
on"
)
rows = rv.get_json()["rows"]
assert len(rows) == 3
def test_envvar_db(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
csv_file = tmp_path / "custom.csv"
csv_file.write_text("timestamp,event,value,user\n2024-01-01 00:00:00,login,5
,bob\n")
monkeypatch.setenv("SCUBADUCK_DB", str(csv_file))
app = server.create_app()
client = app.test_client()
payload = _make_payload()
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
rows = rv.get_json()["rows"]
assert len(rows) == 1
def test_envvar_db_missing(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> N
one:
missing = tmp_path / "missing.sqlite"
monkeypatch.setenv("SCUBADUCK_DB", str(missing))
with pytest.raises(FileNotFoundError):
server.create_app()
def test_group_by_table() -> None:
app = server.app
client = app.test_client()
payload = {
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "table",
"order_by": "user",
"limit": 10,
"columns": ["value"],
"group_by": ["user"],
"aggregate": "Sum",
"show_hits": True,
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I’ll add a test for multiple tables around line 200, after environment variable tests.
py
from __future__ import annotations
import json
from pathlib import Path
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I'll go ahead and patch the code now, as the json import is confirmed.
> *** Begin Patch
> *** Update File: tests/test_server.py
> @@
> def test_envvar_db_missing(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -
> None:
> @@
> server.create_app()
> +
> +
> +def test_multiple_tables(tmp_path: Path) -> None:
> + db_file = tmp_path / "multi.duckdb"
> + con = duckdb.connect(db_file)
> + con.execute("CREATE TABLE t1 (a INT)")
> + con.execute("INSERT INTO t1 VALUES (1)")
> + con.execute("CREATE TABLE t2 (a INT)")
> + con.execute("INSERT INTO t2 VALUES (2)")
> + con.close() # pyright: ignore[reportUnknownMemberType, reportAttributeAc
cessIssue]
> +
> + app = server.create_app(db_file)
> + client = app.test_client()
> + rv = client.get("/api/tables")
> + assert set(rv.get_json()) == {"t1", "t2"}
> + rv = client.get("/api/columns?table=t2")
> + cols = [c["name"] for c in rv.get_json()]
> + assert cols == ["a"]
> + payload = {"table": "t2", "columns": ["a"], "limit": 10}
> + rv = client.post("/api/query", data=json.dumps(payload), content_type="ap
plication/json")
> + assert rv.get_json()["rows"] == [(2,)]
> *** End Patch
> PATCH
Success. Updated the following files:
M tests/test_server.py
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I need to check if any tests manually build payloads without including the table.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# grep -n '"columns":' -n test
s/test_server.py | head
20: "columns": ["timestamp", "event", "value", "user"],
51: "columns": ["timestamp", "event", "value", "user"],
73: "columns": ["timestamp", "event", "value", "user"],
100: "columns": ["timestamp", "user"],
121: "columns": ["timestamp", "event", "value", "user"],
160: "columns": ["timestamp", "event", "value", "user"],
241: payload = {"table": "t2", "columns": ["a"], "limit": 10}
255: "columns": ["value"],
289: "columns": ["timestamp", "event", "value", "user"],
309: "columns": ["timestamp"],
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I need to check where the 'table' entry is missing and add 'table': 'events'.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# sed -n '10,40p' tests/test_s
erver.py
def test_basic_query() -> None:
app = server.app
client = app.test_client()
payload = {
"start": "2024-01-01 00:00:00",
"end": "2024-01-02 00:00:00",
"order_by": "timestamp",
"order_dir": "ASC",
"limit": 10,
"columns": ["timestamp", "event", "value", "user"],
"filters": [],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert data
rows = data["rows"]
# We expect first three rows (until 2024-01-02 00:00:00)
assert len(rows) == 3
assert rows[0][1] == "login"
assert rows[1][1] == "logout"
def test_js_served() -> None:
app = server.app
client = app.test_client()
rv = client.get("/js/chip_input.js")
assert rv.status_code == 200
assert b"initChipInput" in rv.data
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I’ll look for JSON payload definitions and add 'table': 'events' where needed.
ts/test_server.py
14: payload = {
46: payload = {
69: base_payload = {
95: payload = {
241: payload = {"table": "t2", "columns": ["a"], "limit": 10}
249: payload = {
284: payload = {
304: payload = {
323: payload = {
341: payload = {
363: payload = {
379: payload = {
405: payload = {
427: payload = {
451: payload = {
471: payload = {
490: payload = {
511: payload = {
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I'll patch using sed or apply_patch, adding 'table': between lines 12-30.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# sed -n '12,26p' tests/test_s
erver.py
app = server.app
client = app.test_client()
payload = {
"start": "2024-01-01 00:00:00",
"end": "2024-01-02 00:00:00",
"order_by": "timestamp",
"order_dir": "ASC",
"limit": 10,
"columns": ["timestamp", "event", "value", "user"],
"filters": [],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I'll add 'table': 'events' to the snippet now.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# apply_patch<<'PATCH'
> *** Begin Patch
> *** Update File: tests/test_server.py
> @@
> payload = {
> + "table": "events",
> "start": "2024-01-01 00:00:00",
> *** End Patch
> PATCH
Success. Updated the following files:
M tests/test_server.py
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I'll check the second occurrence around line 46 to keep this on track.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# sed -n '44,60p' tests/test_s
erver.py
def test_filter_multi_token() -> None:
app = server.app
client = app.test_client()
payload = {
"start": "2024-01-01 00:00:00",
"end": "2024-01-02 03:00:00",
"order_by": "timestamp",
"limit": 10,
"columns": ["timestamp", "event", "value", "user"],
"filters": [{"column": "user", "op": "=", "value": ["alice", "charlie"]}
],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert data
rows = data["rows"]
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I'll add a new table called 'events' to the database schema.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# apply_patch<<'PATCH'
> *** Begin Patch
> *** Update File: tests/test_server.py
> @@
> payload = {
> + "table": "events",
> "start": "2024-01-01 00:00:00",
> *** End Patch
> PATCH
Success. Updated the following files:
M tests/test_server.pyserver.py
def test_empty_filter_is_noop() -> None:
app = server.app
client = app.test_client()
base_payload = {
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"limit": 100,
"columns": ["timestamp", "event", "value", "user"],
}
no_filter = {**base_payload, "filters": []}
empty_filter = {
**base_payload,
"filters": [{"column": "user", "op": "=", "value": None}],
}
rv1 = client.post(
"/api/query", data=json.dumps(no_filter), content_type="application/json
"
)
rv2 = client.post(
"/api/query", data=json.dumps(empty_filter), content_type="application/j
son"
)
rows1 = rv1.get_json()["rows"]
rows2 = rv2.get_json()["rows"]
assert rows1 == rows2
def test_select_columns() -> None:
app = server.app
client = app.test_client()
payload = {
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"order_by": "timestamp",
"limit": 10,
"columns": ["timestamp", "user"],
"filters": [],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert data
rows = data["rows"]
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I'll add 'table': 'events' to the base and later payloads. Let's do this!
> *** Begin Patch
> *** Update File: tests/test_server.py
> @@
> base_payload = {
> + "table": "events",
> "start": "2024-01-01 00:00:00",
> *** End Patch
> PATCH
Success. Updated the following files:
M tests/test_server.py
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I'll go ahead and patch all the following payload = { lines too.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# grep -n "payload = {" -n tes
ts/test_server.py
14: payload = {
47: payload = {
71: base_payload = {
98: payload = {
244: payload = {"table": "t2", "columns": ["a"], "limit": 10}
252: payload = {
287: payload = {
307: payload = {
326: payload = {
344: payload = {
366: payload = {
382: payload = {
408: payload = {
430: payload = {
454: payload = {
474: payload = {
493: payload = {
514: payload = {
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I'll insert "table": "events", after each "payload = {", using sed.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# perl -0pi -e 's/payload = \{
\n/payload = {\n "table": "events",\n/' tests/test_server.py(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# grep -n "table": "events" -n
tests/test_server.py | head
grep: events: No such file or directory
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#Looks like quoting isn't working as expected. I'll open some lines to check.
bash: .venv/bin/activate: No such file or directory
root@3fe4fa1f8226:/workspace# cd scubaduck
root@3fe4fa1f8226:/workspace/scubaduck# source .venv/bin/activate(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# grep -n "table" -n tests/tes
t_server.py | head
15: "table": "events",
16: "table": "events",
49: "table": "events",
73: "table": "events",
158: "table": "events",
229:def test_multiple_tables(tmp_path: Path) -> None:
240: rv = client.get("/api/tables")
242: rv = client.get("/api/columns?table=t2")
245: payload = {"table": "t2", "columns": ["a"], "limit": 10}
250:def test_group_by_table() -> None:
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#Looks like sed added duplicates in several places like _make_payload. I'll check around line 49.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# sed -n '10,30p' tests/test_s
erver.py
def test_basic_query() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-02 00:00:00",
"order_by": "timestamp",
"order_dir": "ASC",
"limit": 10,
"columns": ["timestamp", "event", "value", "user"],
"filters": [],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert data
rows = data["rows"]
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#Looks like sed introduced duplicates. I'll revert the file and patch manually.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# git checkout -- tests/test_s
erver.py
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I’ll address the missing modifications by manually patching them one step at a time.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# sed -n '140,240p' tests/test
_server.py
)
rows = rv.get_json()["rows"]
assert all(r[3].startswith("a") for r in rows)
assert len(rows) == 2
not_empty = {**base, "filters": [{"column": "user", "op": "!empty"}]}
rv = client.post(
"/api/query", data=json.dumps(not_empty), content_type="application/json
"
)
assert len(rv.get_json()["rows"]) == 4
def _make_payload() -> dict[str, object]:
return {
"start": "2024-01-01 00:00:00",
"end": "2024-01-02 00:00:00",
"order_by": "timestamp",
"order_dir": "ASC",
"limit": 10,
"columns": ["timestamp", "event", "value", "user"],
"filters": [],
}
def test_database_types(tmp_path: Path) -> None:
csv_file = tmp_path / "events.csv"
csv_file.write_text(Path("scubaduck/sample.csv").read_text())
sqlite_file = tmp_path / "events.sqlite"
import sqlite3
conn = sqlite3.connect(sqlite_file)
conn.execute(
"CREATE TABLE events (timestamp TEXT, event TEXT, value INTEGER, user TE
XT)"
)
with open(csv_file) as f:
next(f)
for line in f:
ts, ev, val, user = line.strip().split(",")
conn.execute(
"INSERT INTO events VALUES (?, ?, ?, ?)", (ts, ev, int(val), use
r)
)
conn.commit()
conn.close() # pyright: ignore[reportUnknownMemberType, reportAttributeAcce
ssIssue]
duckdb_file = tmp_path / "events.duckdb"
con = duckdb.connect(duckdb_file)
con.execute(
f"CREATE TABLE events AS SELECT * FROM read_csv_auto('{csv_file.as_posix
()}')"
)
con.close() # pyright: ignore[reportUnknownMemberType, reportAttributeAcces
sIssue]
for db in (csv_file, sqlite_file, duckdb_file):
app = server.create_app(db)
client = app.test_client()
payload = _make_payload()
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/js
on"
)
rows = rv.get_json()["rows"]
assert len(rows) == 3
def test_envvar_db(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
csv_file = tmp_path / "custom.csv"
csv_file.write_text("timestamp,event,value,user\n2024-01-01 00:00:00,login,5
,bob\n")
monkeypatch.setenv("SCUBADUCK_DB", str(csv_file))
app = server.create_app()
client = app.test_client()
payload = _make_payload()
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
rows = rv.get_json()["rows"]
assert len(rows) == 1
def test_envvar_db_missing(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> N
one:
missing = tmp_path / "missing.sqlite"
monkeypatch.setenv("SCUBADUCK_DB", str(missing))
with pytest.raises(FileNotFoundError):
server.create_app()
def test_group_by_table() -> None:
app = server.app
client = app.test_client()
payload = {
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "table",
"order_by": "user",
"limit": 10,
"columns": ["value"],
"group_by": ["user"],
"aggregate": "Sum",
"show_hits": True,
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I'll apply the patch in steps: first, update _make_payload and then adjust all payload dictionaries.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# grep -n "payload = {" -n tes
ts/test_server.py
14: payload = {
46: payload = {
69: base_payload = {
95: payload = {
227: payload = {
262: payload = {
282: payload = {
301: payload = {
319: payload = {
341: payload = {
357: payload = {
383: payload = {
405: payload = {
429: payload = {
449: payload = {
468: payload = {
489: payload = {
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I'll use sed with a unique approach to add 'table': 'events' to each.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# perl -0pi -e 's/payload = \{
\n/payload = {\n "table": "events",\n/' tests/test_server.py
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#Let's review the first few lines and see what's going on.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# sed -n '12,20p' tests/test_s
erver.py
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-02 00:00:00",
"order_by": "timestamp",
"order_dir": "ASC",
"limit": 10,
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I think I’ll need to insert a line just around line 46.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# sed -n '44,52p' tests/test_s
erver.py
def test_filter_multi_token() -> None:
app = server.app
client = app.test_client()
payload = {
"start": "2024-01-01 00:00:00",
"end": "2024-01-02 03:00:00",
"order_by": "timestamp",
"limit": 10,
"columns": ["timestamp", "event", "value", "user"],
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I’ll re-run the patch to check if sedation properly captures the pattern with correct indentation.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# perl -0pi -e 's/(payload = \
{\n)/$1 "table": "events",\n/sg' tests/test_server.py(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# sed -n '44,54p' tests/test_s
erver.py
def test_filter_multi_token() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-02 03:00:00",
"order_by": "timestamp",
"limit": 10,
"columns": ["timestamp", "event", "value", "user"],
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I just confirmed that the base_payload is good to go!
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# sed -n '66,78p' tests/test_s
erver.py
assert rows[-1][3] == "charlie"
def test_empty_filter_is_noop() -> None:
app = server.app
client = app.test_client()
base_payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"limit": 100,
"columns": ["timestamp", "event", "value", "user"],
}
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#Next, I'll look at the following sections for more information.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# grep -n "payload = {" -n tes
ts/test_server.py | tail
309: payload = {
328: payload = {
351: payload = {
368: payload = {
395: payload = {
418: payload = {
443: payload = {
464: payload = {
484: payload = {
506: payload = {
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I’ll review the lines around here to verify the details.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# sed -n '300,315p' tests/test
_server.py
)
data = rv.get_json()
assert rv.status_code == 400
assert "error" in data
def test_query_error_returns_sql_and_traceback() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"columns": ["event"],
"group_by": ["user"],
"aggregate": "avg",
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I’m going to scroll down near the 500-line mark to check.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# sed -n '478,510p' tests/test
_server.py
assert data["rows"][0][1] == 20
def test_timeseries_derived_column() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "timeseries",
"granularity": "1 hour",
"limit": 7,
"columns": ["value"],
"derived_columns": {"derived_1": "value * 2"},
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
rows = data["rows"]
assert all(r[2] == r[1] * 2 for r in rows)
def test_default_start_end_returned() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"order_by": "timestamp",
"limit": 5,
"columns": ["timestamp"],
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#Time to update _make_payload to include the new table field. Let's get that done!
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# sed -n '148,160p' tests/test
_server.py
assert len(rows) == 2
not_empty = {**base, "filters": [{"column": "user", "op": "!empty"}]}
rv = client.post(
"/api/query", data=json.dumps(not_empty), content_type="application/json
"
)
assert len(rv.get_json()["rows"]) == 4
def _make_payload() -> dict[str, object]:
return {
"start": "2024-01-01 00:00:00",
"end": "2024-01-02 00:00:00",(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# sed -n '160,168p' tests/test
_server.py
"end": "2024-01-02 00:00:00",
"order_by": "timestamp",
"order_dir": "ASC",
"limit": 10,
"columns": ["timestamp", "event", "value", "user"],
"filters": [],
}
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I'll place "table": "events" at the top of the return dictionary.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# apply_patch<<'PATCH'
> *** Begin Patch
> *** Update File: tests/test_server.py
> @@
> def _make_payload() -> dict[str, object]:
> return {
> + "table": "events",
> "start": "2024-01-01 00:00:00",
> *** End Patch
> PATCH
Success. Updated the following files:
M tests/test_server.py
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I'll check near the bottom to confirm the last update is applied.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# grep -n "table": tests/test_
server.py | head(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# grep -n '"table":' -n tests/
test_server.py | head
15: "table": "events",
16: "table": "events",
49: "table": "events",
73: "table": "events",
100: "table": "events",
159: "table": "events",
234: "table": "events",
270: "table": "events",
291: "table": "events",
311: "table": "events",
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#Looking at lines 14-18, I suspect sedation added the duplicate "table" lines.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# sed -n '12,20p' tests/test_s
erver.py
app = server.app
client = app.test_client()
payload = {
"table": "events",
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-02 00:00:00",
"order_by": "timestamp",
"order_dir": "ASC",
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#Hmm, I found a duplicate. I'll go ahead and remove it.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# sed -n '14,18p' tests/test_s
erver.py
payload = {
"table": "events",
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-02 00:00:00",
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I’ll go ahead and remove that line now.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# apply_patch<<'PATCH'
> *** Begin Patch
> *** Update File: tests/test_server.py
> @@
> - payload = {
> - "table": "events",
> - "table": "events",
> + payload = {
> + "table": "events",
> *** End Patch
> PATCH
Success. Updated the following files:
M tests/test_server.py(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# sed -n '12,20p' tests/test_s
erver.py
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-02 00:00:00",
"order_by": "timestamp",
"order_dir": "ASC",
"limit": 10,
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I'll scan the other sections to ensure no duplicates are present.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# grep -n '"table": "events"'
-n tests/test_server.py | head
15: "table": "events",
48: "table": "events",
72: "table": "events",
99: "table": "events",
158: "table": "events",
233: "table": "events",
269: "table": "events",
290: "table": "events",
310: "table": "events",
329: "table": "events",
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I’ll apply ruff formatting to tidy up the code now.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# ruff format scubaduck/server
.py scubaduck/static/index.html tests/test_server.py
error: Failed to parse scubaduck/static/index.html:1:1: Expected a statement
1 file reformatted, 1 file left unchanged
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#Ruff format doesn't work for HTML, so I'll focus only on Python as instructed.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# ruff format scubaduck/server
.py tests/test_server.py
2 files left unchanged
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#bash: .venv/bin/activate: No such file or directory
root@3fe4fa1f8226:/workspace# cd scubaduck
root@3fe4fa1f8226:/workspace/scubaduck# source .venv/bin/activate(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# ruff check scubaduck/server.
py tests/test_server.py
scubaduck/server.py:12:8: F401 [*] `sqlite3` imported but unused
|
10 | from pathlib import Path
11 | import os
12 | import sqlite3
| ^^^^^^^ F401
13 | import traceback
|
= help: Remove unused import: `sqlite3`
Found 1 error.
[*] 1 fixable with the `--fix` option.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I'm removing the unused sqlite3 import from server.py since it's no longer needed.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# sed -n '10,20p' scubaduck/se
rver.py
from pathlib import Path
import os
import sqlite3
import traceback
import duckdb
from dateutil import parser as dtparser
from flask import Flask, jsonify, request, send_from_directory
@dataclass
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I'll delete the sqlite3 import since it's no longer needed after the change.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# apply_patch<<'PATCH'
> *** Begin Patch
> *** Update File: scubaduck/server.py
> @@
> -from pathlib import Path
> -import os
> -import sqlite3
> +from pathlib import Path
> +import os
> *** End Patch
> PATCH
Success. Updated the following files:
M scubaduck/server.py
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I'll try running ruff format once more to see if it resolves the issue.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# ruff format scubaduck/server
.py tests/test_server.py
2 files left unchanged
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#Let's run the ruff check once more to ensure everything's good.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# ruff check scubaduck/server.
py tests/test_server.py
All checks passed!(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# pyright0 errors, 0 warnings, 0 informations
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#Time to execute pytest and check if everything's working as expected!
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# pytest -qbringing up nodes......................F.................................................. [100%]
=================================== FAILURES ===================================
_____________________________ test_database_types ______________________________
[gw2] linux -- Python 3.12.3 /workspace/scubaduck/.venv/bin/python
Traceback (most recent call last):
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/runner.p
y", line 341, in from_call
result: TResult | None = func()
^^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/runner.p
y", line 242, in <lambda>
lambda: runtest_hook(item=item, **kwds), when=when, reraise=reraise
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_hooks.py
", line 512, in __call__
return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult
)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_manager.
py", line 120, in _hookexec
return self._inner_hookexec(hook_name, methods, kwargs, firstresult)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_callers.
py", line 167, in _multicall
raise exception
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_callers.
py", line 139, in _multicall
teardown.throw(exception)
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/threadex
ception.py", line 92, in pytest_runtest_call
yield from thread_exception_runtest_hook()
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/threadex
ception.py", line 68, in thread_exception_runtest_hook
yield
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_callers.
py", line 139, in _multicall
teardown.throw(exception)
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/unraisab
leexception.py", line 95, in pytest_runtest_call
yield from unraisable_exception_runtest_hook()
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/unraisab
leexception.py", line 70, in unraisable_exception_runtest_hook
yield
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_callers.
py", line 139, in _multicall
teardown.throw(exception)
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/logging.
py", line 846, in pytest_runtest_call
yield from self._runtest_for(item, "call")
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/logging.
py", line 829, in _runtest_for
yield
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_callers.
py", line 139, in _multicall
teardown.throw(exception)
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/capture.
py", line 898, in pytest_runtest_call
return (yield)
^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_callers.
py", line 139, in _multicall
teardown.throw(exception)
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/skipping
.py", line 257, in pytest_runtest_call
return (yield)
^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_callers.
py", line 121, in _multicall
res = hook_impl.function(*args)
^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/runner.p
y", line 174, in pytest_runtest_call
item.runtest()
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/python.p
y", line 1627, in runtest
self.ihook.pytest_pyfunc_call(pyfuncitem=self)
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_hooks.py
", line 512, in __call__
return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult
)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_manager.
py", line 120, in _hookexec
return self._inner_hookexec(hook_name, methods, kwargs, firstresult)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_callers.
py", line 167, in _multicall
raise exception
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_callers.
py", line 121, in _multicall
res = hook_impl.function(*args)
^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/python.p
y", line 159, in pytest_pyfunc_call
result = testfunction(**testargs)
^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/scubaduck/tests/test_server.py", line 198, in test_database_t
ypes
app = server.create_app(db)
^^^^^^^^^^^^^^^^^^^^^
File "/workspace/scubaduck/scubaduck/server.py", line 264, in create_app
con = _load_database(db_path)
^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/scubaduck/scubaduck/server.py", line 58, in _load_database
con.execute("INSTALL sqlite")
duckdb.duckdb.IOException: IO Error: Failed to download extension "sqlite_scanne
r" at URL "http://extensions.duckdb.org/v1.2.2/linux_amd64_gcc4/sqlite_scanner.d
uckdb_extension.gz"
Extension "sqlite_scanner" is an existing extension.
For more info, visit https://duckdb.org/docs/extensions/troubleshooting/?version
=v1.2.2&platform=linux_amd64_gcc4&extension=sqlite_scanner (ERROR Could not esta
blish connection)
=========================== short test summary info ============================
FAILED tests/test_server.py::test_database_types - duckdb.duckdb.IOException: IO
Error: Failed to download extension "sqlite_s...
1 failed, 69 passed in 12.29s
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#The issue comes from DuckDB trying to install the sqlite extension when offline. I'll remove INSTALL to avoid network dependence.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# sed -n '55,69p' scubaduck/se
rver.py
)
elif ext in {".db", ".sqlite"}:
con = duckdb.connect()
con.execute("INSTALL sqlite")
con.execute("LOAD sqlite")
con.execute(f"ATTACH '{path.as_posix()}' AS db (TYPE SQLITE)")
tables = [
r[0]
for r in con.execute(
"SELECT name FROM db.sqlite_master WHERE type='table'"
).fetchall()
]
for t in tables:
con.execute(f'CREATE VIEW "{t}" AS SELECT * FROM db."{t}"')
else:
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I'll delete the INSTALL line from the configuration.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# apply_patch<<'PATCH'
> *** Begin Patch
> *** Update File: scubaduck/server.py
> @@
> - con = duckdb.connect()
> - con.execute("INSTALL sqlite")
> - con.execute("LOAD sqlite")
> + con = duckdb.connect()
> + con.execute("LOAD sqlite")
> *** End Patch
> PATCH
Success. Updated the following files:
M scubaduck/server.py(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# ruff format scubaduck/server
.py tests/test_server.py
2 files left unchanged(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# ruff check scubaduck/server.
py tests/test_server.py
All checks passed!(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# pyright0 errors, 0 warnings, 0 informations(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# pytest -q
bringing up nodes......................F.................................................. [100%]
=================================== FAILURES ===================================
_____________________________ test_database_types ______________________________
[gw2] linux -- Python 3.12.3 /workspace/scubaduck/.venv/bin/python
Traceback (most recent call last):
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/runner.p
y", line 341, in from_call
result: TResult | None = func()
^^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/runner.p
y", line 242, in <lambda>
lambda: runtest_hook(item=item, **kwds), when=when, reraise=reraise
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_hooks.py
", line 512, in __call__
return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult
)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_manager.
py", line 120, in _hookexec
return self._inner_hookexec(hook_name, methods, kwargs, firstresult)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_callers.
py", line 167, in _multicall
raise exception
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_callers.
py", line 139, in _multicall
teardown.throw(exception)
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/threadex
ception.py", line 92, in pytest_runtest_call
yield from thread_exception_runtest_hook()
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/threadex
ception.py", line 68, in thread_exception_runtest_hook
yield
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_callers.
py", line 139, in _multicall
teardown.throw(exception)
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/unraisab
leexception.py", line 95, in pytest_runtest_call
yield from unraisable_exception_runtest_hook()
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/unraisab
leexception.py", line 70, in unraisable_exception_runtest_hook
yield
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_callers.
py", line 139, in _multicall
teardown.throw(exception)
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/logging.
py", line 846, in pytest_runtest_call
yield from self._runtest_for(item, "call")
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/logging.
py", line 829, in _runtest_for
yield
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_callers.
py", line 139, in _multicall
teardown.throw(exception)
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/capture.
py", line 898, in pytest_runtest_call
return (yield)
^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_callers.
py", line 139, in _multicall
teardown.throw(exception)
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/skipping
.py", line 257, in pytest_runtest_call
return (yield)
^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_callers.
py", line 121, in _multicall
res = hook_impl.function(*args)
^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/runner.p
y", line 174, in pytest_runtest_call
item.runtest()
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/python.p
y", line 1627, in runtest
self.ihook.pytest_pyfunc_call(pyfuncitem=self)
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_hooks.py
", line 512, in __call__
return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult
)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_manager.
py", line 120, in _hookexec
return self._inner_hookexec(hook_name, methods, kwargs, firstresult)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_callers.
py", line 167, in _multicall
raise exception
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/pluggy/_callers.
py", line 121, in _multicall
res = hook_impl.function(*args)
^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/scubaduck/.venv/lib/python3.12/site-packages/_pytest/python.p
y", line 159, in pytest_pyfunc_call
result = testfunction(**testargs)
^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/scubaduck/tests/test_server.py", line 198, in test_database_t
ypes
app = server.create_app(db)
^^^^^^^^^^^^^^^^^^^^^
File "/workspace/scubaduck/scubaduck/server.py", line 263, in create_app
con = _load_database(db_path)
^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/scubaduck/scubaduck/server.py", line 58, in _load_database
con.execute("LOAD sqlite")
duckdb.duckdb.IOException: IO Error: Failed to download extension "sqlite_scanne
r" at URL "http://extensions.duckdb.org/v1.2.2/linux_amd64_gcc4/sqlite_scanner.d
uckdb_extension.gz"
Extension "sqlite_scanner" is an existing extension.
For more info, visit https://duckdb.org/docs/extensions/troubleshooting/?version
=v1.2.2&platform=linux_amd64_gcc4&extension=sqlite_scanner (ERROR Could not esta
blish connection)
=========================== short test summary info ============================
FAILED tests/test_server.py::test_database_types - duckdb.duckdb.IOException: IO
Error: Failed to download extension "sqlite_s...
1 failed, 69 passed in 12.10s
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#It seems the SQLite extension fails to load due to network issues. I can't use online methods, so I'll revert to the simpler sqlite3 method for testing.
Since network is down, I'll bypass installation by preloading SQLite with local path for tests.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# python - <<'PY'
> import duckdb, os, pathlib
> con=duckdb.connect()
> try:
> con.execute('LOAD sqlite')
> print('loaded')
> except Exception as e:
> print('error', e)
> PYerror IO Error: Failed to download extension "sqlite_scanner" at URL "http://ext
ensions.duckdb.org/v1.2.2/linux_amd64_gcc4/sqlite_scanner.duckdb_extension.gz"
Extension "sqlite_scanner" is an existing extension.
For more info, visit https://duckdb.org/docs/extensions/troubleshooting/?version
=v1.2.2&platform=linux_amd64_gcc4&extension=sqlite_scanner (ERROR Could not esta
blish connection)
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#The LOAD sqlite fails to download; I'll check for built-in local extensions instead.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# ls .venv/lib/python*/site-pa
ckages/duckdb | head
__init__.py
__pycache__
bytes_io_wrapper.py
duckdb.cpython-312-x86_64-linux-gnu.so
experimental
filesystem.py
functional
query_graph
typing
udf.py
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#The current extension doesn't work offline, so I'll implement a fallback to manual import.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# sed -n '52,69p' scubaduck/se
rver.py
con = duckdb.connect()
con.execute(
f"CREATE TABLE events AS SELECT * FROM read_csv_auto('{path.as_posix
()}')"
)
elif ext in {".db", ".sqlite"}:
con = duckdb.connect()
con.execute("LOAD sqlite")
con.execute(f"ATTACH '{path.as_posix()}' AS db (TYPE SQLITE)")
tables = [
r[0]
for r in con.execute(
"SELECT name FROM db.sqlite_master WHERE type='table'"
).fetchall()
]
for t in tables:
con.execute(f'CREATE VIEW "{t}" AS SELECT * FROM db."{t}"')
else:
con = duckdb.connect(path)
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I'll wrap it in a try-except block to handle any potential exceptions.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# apply_patch<<'PATCH'
> *** Begin Patch
> *** Update File: scubaduck/server.py
> @@
> elif ext in {".db", ".sqlite"}:
> - con = duckdb.connect()
> - con.execute("LOAD sqlite")
> - con.execute(f"ATTACH '{path.as_posix()}' AS db (TYPE SQLITE)")
> - tables = [
> - r[0]
> - for r in con.execute(
> - "SELECT name FROM db.sqlite_master WHERE type='table'"
> - ).fetchall()
> - ]
> - for t in tables:
> - con.execute(f'CREATE VIEW "{t}" AS SELECT * FROM db."{t}"')
> + con = duckdb.connect()
> + try:
> + con.execute("LOAD sqlite")
> + con.execute(f"ATTACH '{path.as_posix()}' AS db (TYPE SQLITE)")
> + tables = [
> + r[0]
> + for r in con.execute(
> + "SELECT name FROM db.sqlite_master WHERE type='table'"
> + ).fetchall()
> + ]
> + for t in tables:
> + con.execute(f'CREATE VIEW "{t}" AS SELECT * FROM db."{t}"')
> + except Exception:
> + sconn = sqlite3.connect(path)
> + tables = [r[0] for r in sconn.execute(
> + "SELECT name FROM sqlite_master WHERE type='table'"
> + ).fetchall()]
> + for t in tables:
> + info = sconn.execute(f'PRAGMA table_info("{t}")').fetchall()
> + col_defs = ", ".join(f"{r[1]} {r[2]}" for r in info)
> + con.execute(f'CREATE TABLE "{t}" ({col_defs})')
> + placeholders = ",".join("?" for _ in info)
> + for row in sconn.execute(f'SELECT * FROM "{t}"'):
> + con.execute(f'INSERT INTO "{t}" VALUES ({placeholders})',
row)
> + sconn.close()
> *** End Patch
> PATCH
Success. Updated the following files:
M scubaduck/server.py
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I'll try running the ruff formatter once more to check for any fixes.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# ruff format scubaduck/server
.py tests/test_server.py
1 file reformatted, 1 file left unchanged(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# ruff check scubaduck/server.
py tests/test_server.py
scubaduck/server.py:70:21: F821 Undefined name `sqlite3`
|
68 | con.execute(f'CREATE VIEW "{t}" AS SELECT * FROM db."{t}"')
69 | except Exception:
70 | sconn = sqlite3.connect(path)
| ^^^^^^^ F821
71 | tables = [
72 | r[0]
|
Found 1 error.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#Since the fallback now uses sqlite3, I'll add the import back at the top.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# sed -n '10,18p' scubaduck/se
rver.py
from pathlib import Path
import os
import traceback
import duckdb
from dateutil import parser as dtparser
from flask import Flask, jsonify, request, send_from_directory
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I'll reintroduce the import sqlite3 statement into the code.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# apply_patch<<'PATCH'
> *** Begin Patch
> *** Update File: scubaduck/server.py
> @@
> -from pathlib import Path
> -import os
> -import traceback
> +from pathlib import Path
> +import os
> +import sqlite3
> +import traceback
> *** End Patch
> PATCH
Success. Updated the following files:
M scubaduck/server.py(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# ruff format scubaduck/server
.py tests/test_server.py
2 files left unchanged(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# ruff check scubaduck/server.
py tests/test_server.py
All checks passed!
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I'll go ahead and try running Pyright once again.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# pyright0 errors, 0 warnings, 0 informations(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# pytest -q
bringing up nodes......................................................................... [100%]
70 passed in 11.57s
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#Finally, all tests passed! Next, I’ll verify the status in Git.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# git status --short
M scubaduck/server.py
M scubaduck/static/index.html
M tests/test_server.py
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I'll check index.html to make sure all the file modifications are properly added.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# git diff --stat
scubaduck/server.py | 87 ++++++++++++----
scubaduck/static/index.html | 240 +++++++++++++++++++++++++-------------------
tests/test_server.py | 18 ++++
3 files changed, 222 insertions(+), 123 deletions(-)
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I'll review index.html briefly and then finish by committing the changes.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# git add scubaduck/server.py
scubaduck/static/index.html tests/test_server.py(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# git commit -m "feat(server):
support multiple tables" && git status --short
[work a035ba8] feat(server): support multiple tables
3 files changed, 222 insertions(+), 123 deletions(-)(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# git status --short
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I'll gather the necessary lines from server.py and index.html for the PR message.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# nl -ba scubaduck/server.py |
sed -n '253,320p'
253 elif op == "!contains":
254 where_parts.append(f"{f.column} NOT ILIKE '%' || {val} || '%
'")
255 elif op == "empty":
256 where_parts.append(f"{f.column} = {val}")
257 elif op == "!empty":
258 where_parts.append(f"{f.column} != {val}")
259 else:
260 where_parts.append(f"{f.column} {op} {val}")
261 if where_parts:
262 query += " WHERE " + " AND ".join(where_parts)
263 if group_cols:
264 query += " GROUP BY " + ", ".join(group_cols)
265 if params.order_by:
266 query += f" ORDER BY {params.order_by} {params.order_dir}"
267 elif params.graph_type == "timeseries":
268 query += " ORDER BY bucket"
269 if params.limit is not None:
270 query += f" LIMIT {params.limit}"
271 return query
272
273
274 def create_app(db_file: str | Path | None = None) -> Flask:
275 app = Flask(__name__, static_folder="static")
276 if db_file is None:
277 env_db = os.environ.get("SCUBADUCK_DB")
278 if env_db:
279 db_file = env_db
280 db_path = Path(db_file or Path(__file__).with_name("sample.csv")).re
solve()
281 con = _load_database(db_path)
282 tables = [r[0] for r in con.execute("SHOW TABLES").fetchall()]
283 if not tables:
284 raise ValueError("No tables found in database")
285 default_table = tables[0]
286 columns_cache: Dict[str, Dict[str, str]] = {}
287
288 def get_columns(table: str) -> Dict[str, str]:
289 if table not in columns_cache:
290 rows = con.execute(f'PRAGMA table_info("{table}")').fetchall
()
291 if not rows:
292 raise ValueError(f"Unknown table: {table}")
293 columns_cache[table] = {r[1]: r[2] for r in rows}
294 return columns_cache[table]
295
296 sample_cache: Dict[Tuple[str, str, str], Tuple[List[str], float]] =
{}
297 CACHE_TTL = 60.0
298 CACHE_LIMIT = 200
299
300 @app.route("/")
301 def index() -> Any: # pyright: ignore[reportUnusedFunction]
302 assert app.static_folder is not None
303 return send_from_directory(app.static_folder, "index.html")
304
305 @app.route("/js/<path:filename>")
306 def js(filename: str) -> Any: # pyright: ignore[reportUnusedFunctio
n]
307 assert app.static_folder is not None
308 folder = Path(app.static_folder) / "js"
309 return send_from_directory(folder, filename)
310
311 @app.route("/api/tables")
312 def tables_endpoint() -> Any: # pyright: ignore[reportUnusedFunctio
n]
313 return jsonify(tables)
314
315 @app.route("/api/columns")
316 def columns() -> Any: # pyright: ignore[reportUnusedFunction]
317 table = request.args.get("table", default_table)
318 rows = con.execute(f'PRAGMA table_info("{table}")').fetchall()
319 return jsonify([{"name": r[1], "type": r[2]} for r in rows])
320
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#I’m adding a new snippet for _load_database.
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# nl -ba scubaduck/server.py |
sed -n '40,80p'
40 show_hits: bool = False
41 x_axis: str | None = None
42 granularity: str = "Auto"
43 fill: str = "0"
44 table: str = "events"
45
46
47 def _load_database(path: Path) -> duckdb.DuckDBPyConnection:
48 if not path.exists():
49 raise FileNotFoundError(path)
50
51 ext = path.suffix.lower()
52 if ext == ".csv":
53 con = duckdb.connect()
54 con.execute(
55 f"CREATE TABLE events AS SELECT * FROM read_csv_auto('{path.
as_posix()}')"
56 )
57 elif ext in {".db", ".sqlite"}:
58 con = duckdb.connect()
59 try:
60 con.execute("LOAD sqlite")
61 con.execute(f"ATTACH '{path.as_posix()}' AS db (TYPE SQLITE)
")
62 tables = [
63 r[0]
64 for r in con.execute(
65 "SELECT name FROM db.sqlite_master WHERE type='table
'"
66 ).fetchall()
67 ]
68 for t in tables:
69 con.execute(f'CREATE VIEW "{t}" AS SELECT * FROM db."{t}
"')
70 except Exception:
71 sconn = sqlite3.connect(path)
72 tables = [
73 r[0]
74 for r in sconn.execute(
75 "SELECT name FROM sqlite_master WHERE type='table'"
76 ).fetchall()
77 ]
78 for t in tables:
79 info = sconn.execute(f'PRAGMA table_info("{t}")').fetcha
ll()
80 col_defs = ", ".join(f"{r[1]} {r[2]}" for r in info)(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# nl -ba scubaduck/static/inde
x.html | sed -n '138,158p'
138 </style>
139 </head>
140 <body>
141 <div id="header">sample.csv - <select id="table"></select> <select id=
"graph_type"><option value="samples">Samples</option><option value="table">Table
</option><option value="timeseries">Time Series</option></select></div>
142 <div id="content">
143 <div id="sidebar">
144 <div id="tabs">
145 <button class="tab active" data-tab="settings">View Settings</bu
tton>
146 <button class="tab" data-tab="columns">Columns</button>
147 <button id="dive" onclick="dive()">Dive</button>
148 </div>
149 <div id="settings" class="tab-content active">
150 <div class="field">
151 <label>Start<span class="help" title="Sets the start/end of th
e time range to query. Can be any kind of datetime string. For example: 'April 2
3, 2014' or 'yesterday'.">[?]</span></label>
152 <div class="rel-box">
153 <input id="start" type="text" />
154 <button type="button" class="rel-btn" data-target="start-sel
ect">▼</button>
155 <div id="start-select" class="rel-dropdown" data-input="star
t">
156 <div data-value="-1 hour">-1 hour</div>
157 <div data-value="-3 hours">-3 hours</div>
158 <div data-value="-12 hours">-12 hours</div>(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck# nl -ba scubaduck/static/inde
x.html | sed -n '456,610p'
456 orderDirBtn.addEventListener('click', () => {
457 orderDir = orderDir === 'ASC' ? 'DESC' : 'ASC';
458 updateOrderDirButton();
459 });
460 updateOrderDirButton();
461 graphTypeSel.addEventListener('change', updateDisplayTypeUI);
462
463 function loadColumns(table) {
464 return fetch('/api/columns?table=' + encodeURIComponent(table)).then(r
=> r.json()).then(cols => {
465 const orderSelect = document.getElementById('order_by');
466 const xAxisSelect = document.getElementById('x_axis');
467 const groupsEl = document.getElementById('column_groups');
468 orderSelect.innerHTML = '';
469 xAxisSelect.innerHTML = '';
470 groupsEl.innerHTML = '';
471 allColumns.length = 0;
472 stringColumns.length = 0;
473 integerColumns.length = 0;
474 timeColumns.length = 0;
475 baseColumns.length = 0;
476 baseStringColumns.length = 0;
477 baseIntegerColumns.length = 0;
478 baseTimeColumns.length = 0;
479 for (const k in columnTypes) delete columnTypes[k];
480 const groups = {
481 time: {name: 'Time', cols: [], ul: null},
482 integer: {name: 'Integers', cols: [], ul: null},
483 string: {name: 'Strings', cols: [], ul: null},
484 };
485 cols.forEach(c => {
486 const t = c.type.toUpperCase();
487 columnTypes[c.name] = c.type;
488 allColumns.push(c.name);
489 baseColumns.push(c.name);
490 let g = 'string';
491 if (t.includes('INT')) g = 'integer';
492 if (t.includes('TIMESTAMP')) {
493 g = 'time';
494 timeColumns.push(c.name);
495 baseTimeColumns.push(c.name);
496 }
497 if (g === 'string') {
498 stringColumns.push(c.name);
499 baseStringColumns.push(c.name);
500 } else {
501 integerColumns.push(c.name);
502 baseIntegerColumns.push(c.name);
503 }
504 groups[g].cols.push(c.name);
505 if (g !== 'string') {
506 const o = document.createElement('option');
507 o.value = c.name;
508 o.textContent = c.name;
509 orderSelect.appendChild(o);
510 }
511 });
512 timeColumns.forEach(name => {
513 const o = document.createElement('option');
514 o.value = name;
515 o.textContent = name;
516 xAxisSelect.appendChild(o);
517 });
518 Object.keys(groups).forEach(key => {
519 const g = groups[key];
520 const div = document.createElement('div');
521 div.className = 'col-group';
522 const header = document.createElement('div');
523 header.className = 'col-group-header';
524 header.appendChild(document.createTextNode(g.name + ': '));
525 const links = document.createElement('span');
526 links.className = 'links';
527 const allBtn = document.createElement('a');
528 allBtn.href = '#';
529 allBtn.textContent = 'All';
530 const noneBtn = document.createElement('a');
531 noneBtn.href = '#';
532 noneBtn.textContent = 'None';
533 links.appendChild(allBtn);
534 links.appendChild(noneBtn);
535 header.appendChild(links);
536 div.appendChild(header);
537 const ul = document.createElement('ul');
538 g.ul = ul;
539 g.cols.forEach(name => {
540 const li = document.createElement('li');
541 const label = document.createElement('label');
542 const cb = document.createElement('input');
543 cb.type = 'checkbox';
544 cb.value = name;
545 cb.checked = true;
546 cb.addEventListener('change', updateSelectedColumns);
547 label.appendChild(cb);
548 label.appendChild(document.createTextNode(' ' + name));
549 li.appendChild(label);
550 ul.appendChild(li);
551 });
552 allBtn.addEventListener('click', e => {
553 e.preventDefault();
554 ul.querySelectorAll('input').forEach(cb => (cb.checked = true));
555 updateSelectedColumns();
556 });
557 noneBtn.addEventListener('click', e => {
558 e.preventDefault();
559 ul.querySelectorAll('input').forEach(cb => (cb.checked = false))
;
560 updateSelectedColumns();
561 });
562 div.appendChild(ul);
563 groupsEl.appendChild(div);
564 });
565 document.getElementById('columns_all').addEventListener('click', e =
> {
566 e.preventDefault();
567 groupsEl.querySelectorAll('input').forEach(cb => (cb.checked = tru
e));
568 updateSelectedColumns();
569 });
570 document.getElementById('columns_none').addEventListener('click', e
=> {
571 e.preventDefault();
572 groupsEl.querySelectorAll('input').forEach(cb => (cb.checked = fal
se));
573 updateSelectedColumns();
574 });
575 updateSelectedColumns();
576 groupBy = document.getElementById('group_by').closest('.field');
577 initChipInput(groupBy, typed =>
578 allColumns.filter(c => c.toLowerCase().includes(typed.toLowerCase(
)))
579 );
580 initDropdown(orderSelect);
581 initDropdown(document.getElementById('aggregate'));
582 });
583 }
584
585 let columnsInitialized = false;
586 fetch('/api/tables').then(r => r.json()).then(tables => {
587 tables.forEach(t => {
588 const o = document.createElement('option');
589 o.value = t;
590 o.textContent = t;
591 document.getElementById('table').appendChild(o);
592 });
593 const table = parseSearch().table || tables[0];
594 document.getElementById('table').value = table;
595 loadColumns(table).then(() => {
596 updateDisplayTypeUI();
597 addFilter();
598 initFromUrl();
599 columnsInitialized = true;
600 });
601 document.getElementById('table').addEventListener('change', () => {
602 loadColumns(document.getElementById('table').value).then(() => {
603 if (columnsInitialized) {
604 applyParams(parseSearch());
605 }
606 });
607 });
608 });
609
610 document.querySelectorAll('#tabs .tab').forEach(btn => {
(scubaduck) root@3fe4fa1f8226:/workspace/scubaduck#