Task ID: task_e_682f2dea67208323a60ab54a67d0c6fd
Building scubaduck @ file:///workspace/scubaduck
⠙ Preparing packages... (0/33)
python-slugify ------------------------------ 8.00 KiB/9.82 KiB
pyee ------------------------------ 15.36 KiB/15.36 KiB
itsdangerous ------------------------------ 8.00 KiB/15.85 KiB
pluggy ------------------------------ 16.00 KiB/20.06 KiB
typing-extensions ------------------------------ 32.00 KiB/44.73 KiB
pytest-xdist ------------------------------ 0 B/45.03 KiB
requests ------------------------------ 8.00 KiB/63.41 KiB
packaging ------------------------------ 0 B/64.91 KiB
idna ------------------------------ 0 B/68.79 KiB
text-unidecode ------------------------------ 16.00 KiB/76.32 KiB
flask ------------------------------ 14.88 KiB/100.88 KiB
urllib3 ------------------------------ 0 B/125.66 KiB
werkzeug ------------------------------ 0 B/219.24 KiB
python-dateutil ------------------------------ 8.00 KiB/224.50 KiB
ruff ------------------------------ 170.95 KiB/11.02 MiB
duckdb ------------------------------ 8.00 KiB/19.27 MiB
playwright ------------------------------ 32.00 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠙ Preparing packages... (0/33)
python-slugify ------------------------------ 8.00 KiB/9.82 KiB
itsdangerous ------------------------------ 8.00 KiB/15.85 KiB
pluggy ------------------------------ 16.00 KiB/20.06 KiB
typing-extensions ------------------------------ 32.00 KiB/44.73 KiB
pytest-xdist ------------------------------ 0 B/45.03 KiB
requests ------------------------------ 8.00 KiB/63.41 KiB
packaging ------------------------------ 0 B/64.91 KiB
idna ------------------------------ 0 B/68.79 KiB
text-unidecode ------------------------------ 16.00 KiB/76.32 KiB
flask ------------------------------ 14.88 KiB/100.88 KiB
urllib3 ------------------------------ 0 B/125.66 KiB
werkzeug ------------------------------ 0 B/219.24 KiB
python-dateutil ------------------------------ 8.00 KiB/224.50 KiB
ruff ------------------------------ 170.95 KiB/11.02 MiB
duckdb ------------------------------ 8.00 KiB/19.27 MiB
playwright ------------------------------ 32.00 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠙ Preparing packages... (0/33)
pytest-base-url ------------------------------ 5.18 KiB/5.18 KiB
iniconfig ------------------------------ 5.91 KiB/5.91 KiB
python-slugify ------------------------------ 9.82 KiB/9.82 KiB
itsdangerous ------------------------------ 15.85 KiB/15.85 KiB
nodeenv ------------------------------ 21.79 KiB/21.79 KiB
typing-extensions ------------------------------ 44.73 KiB/44.73 KiB
pytest-xdist ------------------------------ 16.00 KiB/45.03 KiB
requests ------------------------------ 30.88 KiB/63.41 KiB
packaging ------------------------------ 16.00 KiB/64.91 KiB
idna ------------------------------ 32.00 KiB/68.79 KiB
text-unidecode ------------------------------ 76.32 KiB/76.32 KiB
click ------------------------------ 32.00 KiB/99.76 KiB
flask ------------------------------ 46.88 KiB/100.88 KiB
urllib3 ------------------------------ 48.00 KiB/125.66 KiB
werkzeug ------------------------------ 48.00 KiB/219.24 KiB
python-dateutil ------------------------------ 48.00 KiB/224.50 KiB
pytest ------------------------------ 32.00 KiB/335.58 KiB
greenlet ------------------------------ 0 B/589.71 KiB
pyright ------------------------------ 8.00 KiB/5.31 MiB
ruff ------------------------------ 398.95 KiB/11.02 MiB
duckdb ------------------------------ 80.00 KiB/19.27 MiB
playwright ------------------------------ 48.00 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠙ Preparing packages... (0/33)
pytest-base-url ------------------------------ 5.18 KiB/5.18 KiB
iniconfig ------------------------------ 5.91 KiB/5.91 KiB
python-slugify ------------------------------ 9.82 KiB/9.82 KiB
nodeenv ------------------------------ 21.79 KiB/21.79 KiB
typing-extensions ------------------------------ 44.73 KiB/44.73 KiB
pytest-xdist ------------------------------ 32.00 KiB/45.03 KiB
requests ------------------------------ 30.88 KiB/63.41 KiB
packaging ------------------------------ 16.00 KiB/64.91 KiB
idna ------------------------------ 32.00 KiB/68.79 KiB
text-unidecode ------------------------------ 76.32 KiB/76.32 KiB
click ------------------------------ 48.00 KiB/99.76 KiB
flask ------------------------------ 46.88 KiB/100.88 KiB
urllib3 ------------------------------ 48.00 KiB/125.66 KiB
werkzeug ------------------------------ 48.00 KiB/219.24 KiB
python-dateutil ------------------------------ 48.00 KiB/224.50 KiB
pytest ------------------------------ 32.00 KiB/335.58 KiB
greenlet ------------------------------ 0 B/589.71 KiB
pyright ------------------------------ 8.00 KiB/5.31 MiB
ruff ------------------------------ 414.95 KiB/11.02 MiB
duckdb ------------------------------ 96.00 KiB/19.27 MiB
playwright ------------------------------ 48.00 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠙ Preparing packages... (0/33)
pytest-base-url ------------------------------ 5.18 KiB/5.18 KiB
iniconfig ------------------------------ 5.91 KiB/5.91 KiB
python-slugify ------------------------------ 9.82 KiB/9.82 KiB
nodeenv ------------------------------ 21.79 KiB/21.79 KiB
pytest-xdist ------------------------------ 32.00 KiB/45.03 KiB
requests ------------------------------ 30.88 KiB/63.41 KiB
packaging ------------------------------ 16.00 KiB/64.91 KiB
idna ------------------------------ 32.00 KiB/68.79 KiB
text-unidecode ------------------------------ 76.32 KiB/76.32 KiB
click ------------------------------ 48.00 KiB/99.76 KiB
flask ------------------------------ 46.88 KiB/100.88 KiB
urllib3 ------------------------------ 48.00 KiB/125.66 KiB
werkzeug ------------------------------ 48.00 KiB/219.24 KiB
python-dateutil ------------------------------ 48.00 KiB/224.50 KiB
pytest ------------------------------ 32.00 KiB/335.58 KiB
greenlet ------------------------------ 0 B/589.71 KiB
pyright ------------------------------ 8.00 KiB/5.31 MiB
ruff ------------------------------ 414.95 KiB/11.02 MiB
duckdb ------------------------------ 96.00 KiB/19.27 MiB
playwright ------------------------------ 48.00 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠙ Preparing packages... (0/33)
iniconfig ------------------------------ 5.91 KiB/5.91 KiB
python-slugify ------------------------------ 9.82 KiB/9.82 KiB
nodeenv ------------------------------ 21.79 KiB/21.79 KiB
pytest-xdist ------------------------------ 32.00 KiB/45.03 KiB
requests ------------------------------ 30.88 KiB/63.41 KiB
packaging ------------------------------ 16.00 KiB/64.91 KiB
idna ------------------------------ 32.00 KiB/68.79 KiB
text-unidecode ------------------------------ 76.32 KiB/76.32 KiB
click ------------------------------ 48.00 KiB/99.76 KiB
flask ------------------------------ 46.88 KiB/100.88 KiB
urllib3 ------------------------------ 48.00 KiB/125.66 KiB
werkzeug ------------------------------ 48.00 KiB/219.24 KiB
python-dateutil ------------------------------ 48.00 KiB/224.50 KiB
pytest ------------------------------ 32.00 KiB/335.58 KiB
greenlet ------------------------------ 0 B/589.71 KiB
pyright ------------------------------ 8.00 KiB/5.31 MiB
ruff ------------------------------ 414.95 KiB/11.02 MiB
duckdb ------------------------------ 112.00 KiB/19.27 MiB
playwright ------------------------------ 48.00 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠙ Preparing packages... (0/33)
python-slugify ------------------------------ 9.82 KiB/9.82 KiB
nodeenv ------------------------------ 21.79 KiB/21.79 KiB
pytest-xdist ------------------------------ 32.00 KiB/45.03 KiB
requests ------------------------------ 30.88 KiB/63.41 KiB
packaging ------------------------------ 32.00 KiB/64.91 KiB
idna ------------------------------ 48.00 KiB/68.79 KiB
text-unidecode ------------------------------ 76.32 KiB/76.32 KiB
click ------------------------------ 48.00 KiB/99.76 KiB
flask ------------------------------ 62.88 KiB/100.88 KiB
urllib3 ------------------------------ 48.00 KiB/125.66 KiB
werkzeug ------------------------------ 48.00 KiB/219.24 KiB
python-dateutil ------------------------------ 48.00 KiB/224.50 KiB
pytest ------------------------------ 32.00 KiB/335.58 KiB
greenlet ------------------------------ 0 B/589.71 KiB
pyright ------------------------------ 8.00 KiB/5.31 MiB
ruff ------------------------------ 430.95 KiB/11.02 MiB
duckdb ------------------------------ 128.00 KiB/19.27 MiB
playwright ------------------------------ 48.00 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠙ Preparing packages... (0/33)
python-slugify ------------------------------ 9.82 KiB/9.82 KiB
pytest-xdist ------------------------------ 32.00 KiB/45.03 KiB
requests ------------------------------ 30.88 KiB/63.41 KiB
packaging ------------------------------ 32.00 KiB/64.91 KiB
idna ------------------------------ 48.00 KiB/68.79 KiB
text-unidecode ------------------------------ 76.32 KiB/76.32 KiB
click ------------------------------ 64.00 KiB/99.76 KiB
flask ------------------------------ 62.88 KiB/100.88 KiB
urllib3 ------------------------------ 48.00 KiB/125.66 KiB
werkzeug ------------------------------ 48.00 KiB/219.24 KiB
python-dateutil ------------------------------ 48.00 KiB/224.50 KiB
pytest ------------------------------ 32.00 KiB/335.58 KiB
greenlet ------------------------------ 0 B/589.71 KiB
pyright ------------------------------ 8.00 KiB/5.31 MiB
ruff ------------------------------ 430.95 KiB/11.02 MiB
duckdb ------------------------------ 128.00 KiB/19.27 MiB
playwright ------------------------------ 48.00 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠙ Preparing packages... (0/33)
python-slugify ------------------------------ 9.82 KiB/9.82 KiB
pytest-xdist ------------------------------ 32.00 KiB/45.03 KiB
requests ------------------------------ 30.88 KiB/63.41 KiB
packaging ------------------------------ 32.00 KiB/64.91 KiB
idna ------------------------------ 48.00 KiB/68.79 KiB
click ------------------------------ 64.00 KiB/99.76 KiB
flask ------------------------------ 62.88 KiB/100.88 KiB
urllib3 ------------------------------ 48.00 KiB/125.66 KiB
werkzeug ------------------------------ 62.68 KiB/219.24 KiB
python-dateutil ------------------------------ 48.00 KiB/224.50 KiB
pytest ------------------------------ 32.00 KiB/335.58 KiB
greenlet ------------------------------ 0 B/589.71 KiB
pyright ------------------------------ 16.00 KiB/5.31 MiB
ruff ------------------------------ 430.95 KiB/11.02 MiB
duckdb ------------------------------ 128.00 KiB/19.27 MiB
playwright ------------------------------ 48.00 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠙ Preparing packages... (0/33)
python-slugify ------------------------------ 9.82 KiB/9.82 KiB
pytest-xdist ------------------------------ 32.00 KiB/45.03 KiB
requests ------------------------------ 30.88 KiB/63.41 KiB
packaging ------------------------------ 32.00 KiB/64.91 KiB
idna ------------------------------ 48.00 KiB/68.79 KiB
click ------------------------------ 64.00 KiB/99.76 KiB
flask ------------------------------ 62.88 KiB/100.88 KiB
urllib3 ------------------------------ 48.00 KiB/125.66 KiB
werkzeug ------------------------------ 62.68 KiB/219.24 KiB
python-dateutil ------------------------------ 48.00 KiB/224.50 KiB
pytest ------------------------------ 32.00 KiB/335.58 KiB
greenlet ------------------------------ 0 B/589.71 KiB
pyright ------------------------------ 16.00 KiB/5.31 MiB
ruff ------------------------------ 430.95 KiB/11.02 MiB
duckdb ------------------------------ 128.00 KiB/19.27 MiB
playwright ------------------------------ 60.25 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (15/33)
pytest-xdist ------------------------------ 32.00 KiB/45.03 KiB
requests ------------------------------ 30.88 KiB/63.41 KiB
packaging ------------------------------ 32.00 KiB/64.91 KiB
idna ------------------------------ 64.00 KiB/68.79 KiB
click ------------------------------ 64.00 KiB/99.76 KiB
flask ------------------------------ 62.88 KiB/100.88 KiB
urllib3 ------------------------------ 48.00 KiB/125.66 KiB
werkzeug ------------------------------ 62.68 KiB/219.24 KiB
python-dateutil ------------------------------ 48.00 KiB/224.50 KiB
pytest ------------------------------ 48.00 KiB/335.58 KiB
greenlet ------------------------------ 0 B/589.71 KiB
pyright ------------------------------ 16.00 KiB/5.31 MiB
ruff ------------------------------ 446.95 KiB/11.02 MiB
duckdb ------------------------------ 144.00 KiB/19.27 MiB
playwright ------------------------------ 60.25 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (15/33)
pytest-xdist ------------------------------ 45.03 KiB/45.03 KiB
requests ------------------------------ 46.88 KiB/63.41 KiB
packaging ------------------------------ 48.00 KiB/64.91 KiB
click ------------------------------ 80.00 KiB/99.76 KiB
flask ------------------------------ 78.88 KiB/100.88 KiB
urllib3 ------------------------------ 77.80 KiB/125.66 KiB
werkzeug ------------------------------ 94.68 KiB/219.24 KiB
python-dateutil ------------------------------ 110.50 KiB/224.50 KiB
pytest ------------------------------ 76.68 KiB/335.58 KiB
greenlet ------------------------------ 16.00 KiB/589.71 KiB
pyright ------------------------------ 16.00 KiB/5.31 MiB
ruff ------------------------------ 558.95 KiB/11.02 MiB
duckdb ------------------------------ 208.00 KiB/19.27 MiB
playwright ------------------------------ 76.25 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (15/33)
pytest-xdist ------------------------------ 45.03 KiB/45.03 KiB
packaging ------------------------------ 64.91 KiB/64.91 KiB
click ------------------------------ 99.76 KiB/99.76 KiB
flask ------------------------------ 94.88 KiB/100.88 KiB
urllib3 ------------------------------ 93.80 KiB/125.66 KiB
werkzeug ------------------------------ 94.68 KiB/219.24 KiB
python-dateutil ------------------------------ 222.50 KiB/224.50 KiB
pytest ------------------------------ 124.68 KiB/335.58 KiB
greenlet ------------------------------ 32.00 KiB/589.71 KiB
pyright ------------------------------ 93.35 KiB/5.31 MiB
ruff ------------------------------ 750.95 KiB/11.02 MiB
duckdb ------------------------------ 336.00 KiB/19.27 MiB
playwright ------------------------------ 92.25 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (15/33)
pytest-xdist ------------------------------ 45.03 KiB/45.03 KiB
packaging ------------------------------ 64.91 KiB/64.91 KiB
flask ------------------------------ 94.88 KiB/100.88 KiB
urllib3 ------------------------------ 93.80 KiB/125.66 KiB
werkzeug ------------------------------ 110.68 KiB/219.24 KiB
python-dateutil ------------------------------ 222.50 KiB/224.50 KiB
pytest ------------------------------ 124.68 KiB/335.58 KiB
greenlet ------------------------------ 32.00 KiB/589.71 KiB
pyright ------------------------------ 157.35 KiB/5.31 MiB
ruff ------------------------------ 814.95 KiB/11.02 MiB
duckdb ------------------------------ 384.00 KiB/19.27 MiB
playwright ------------------------------ 92.25 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (15/33)
packaging ------------------------------ 64.91 KiB/64.91 KiB
flask ------------------------------ 100.88 KiB/100.88 KiB
urllib3 ------------------------------ 93.80 KiB/125.66 KiB
werkzeug ------------------------------ 110.68 KiB/219.24 KiB
python-dateutil ------------------------------ 222.50 KiB/224.50 KiB
pytest ------------------------------ 124.68 KiB/335.58 KiB
greenlet ------------------------------ 32.00 KiB/589.71 KiB
pyright ------------------------------ 173.35 KiB/5.31 MiB
ruff ------------------------------ 830.95 KiB/11.02 MiB
duckdb ------------------------------ 400.00 KiB/19.27 MiB
playwright ------------------------------ 92.25 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (15/33)
flask ------------------------------ 100.88 KiB/100.88 KiB
urllib3 ------------------------------ 93.80 KiB/125.66 KiB
werkzeug ------------------------------ 110.68 KiB/219.24 KiB
python-dateutil ------------------------------ 222.50 KiB/224.50 KiB
pytest ------------------------------ 124.68 KiB/335.58 KiB
greenlet ------------------------------ 48.00 KiB/589.71 KiB
pyright ------------------------------ 173.35 KiB/5.31 MiB
ruff ------------------------------ 830.95 KiB/11.02 MiB
duckdb ------------------------------ 416.00 KiB/19.27 MiB
playwright ------------------------------ 92.25 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (15/33)
flask ------------------------------ 100.88 KiB/100.88 KiB
urllib3 ------------------------------ 93.80 KiB/125.66 KiB
werkzeug ------------------------------ 126.68 KiB/219.24 KiB
pytest ------------------------------ 140.68 KiB/335.58 KiB
greenlet ------------------------------ 48.00 KiB/589.71 KiB
pyright ------------------------------ 221.35 KiB/5.31 MiB
ruff ------------------------------ 878.95 KiB/11.02 MiB
duckdb ------------------------------ 464.00 KiB/19.27 MiB
playwright ------------------------------ 108.25 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (15/33)
urllib3 ------------------------------ 109.80 KiB/125.66 KiB
werkzeug ------------------------------ 142.68 KiB/219.24 KiB
pytest ------------------------------ 172.68 KiB/335.58 KiB
greenlet ------------------------------ 61.60 KiB/589.71 KiB
pyright ------------------------------ 333.35 KiB/5.31 MiB
ruff ------------------------------ 990.95 KiB/11.02 MiB
duckdb ------------------------------ 560.00 KiB/19.27 MiB
playwright ------------------------------ 108.25 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (15/33)
urllib3 ------------------------------ 125.66 KiB/125.66 KiB
werkzeug ------------------------------ 142.68 KiB/219.24 KiB
pytest ------------------------------ 204.68 KiB/335.58 KiB
greenlet ------------------------------ 253.60 KiB/589.71 KiB
pyright ------------------------------ 525.35 KiB/5.31 MiB
ruff ------------------------------ 1.17 MiB/11.02 MiB
duckdb ------------------------------ 785.61 KiB/19.27 MiB
playwright ------------------------------ 108.25 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (15/33)
werkzeug ------------------------------ 158.68 KiB/219.24 KiB
pytest ------------------------------ 220.68 KiB/335.58 KiB
greenlet ------------------------------ 381.60 KiB/589.71 KiB
pyright ------------------------------ 669.35 KiB/5.31 MiB
ruff ------------------------------ 1.30 MiB/11.02 MiB
duckdb ------------------------------ 913.61 KiB/19.27 MiB
playwright ------------------------------ 220.25 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (15/33)
pytest ------------------------------ 316.68 KiB/335.58 KiB
greenlet ------------------------------ 477.60 KiB/589.71 KiB
pyright ------------------------------ 1.17 MiB/5.31 MiB
ruff ------------------------------ 1.85 MiB/11.02 MiB
duckdb ------------------------------ 1.45 MiB/19.27 MiB
playwright ------------------------------ 716.25 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (15/33)
pytest ------------------------------ 316.68 KiB/335.58 KiB
greenlet ------------------------------ 477.60 KiB/589.71 KiB
pyright ------------------------------ 1.31 MiB/5.31 MiB
ruff ------------------------------ 1.98 MiB/11.02 MiB
duckdb ------------------------------ 1.56 MiB/19.27 MiB
playwright ------------------------------ 860.25 KiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (15/33)
greenlet ------------------------------ 493.60 KiB/589.71 KiB
pyright ------------------------------ 1.53 MiB/5.31 MiB
ruff ------------------------------ 2.28 MiB/11.02 MiB
duckdb ------------------------------ 1.89 MiB/19.27 MiB
playwright ------------------------------ 1.14 MiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (15/33)
greenlet ------------------------------ 573.60 KiB/589.71 KiB
pyright ------------------------------ 1.80 MiB/5.31 MiB
ruff ------------------------------ 3.11 MiB/11.02 MiB
duckdb ------------------------------ 2.73 MiB/19.27 MiB
playwright ------------------------------ 1.98 MiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (15/33)
pyright ------------------------------ 1.81 MiB/5.31 MiB
ruff ------------------------------ 3.59 MiB/11.02 MiB
duckdb ------------------------------ 3.22 MiB/19.27 MiB
playwright ------------------------------ 2.47 MiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠹ Preparing packages... (15/33)
pyright ------------------------------ 1.84 MiB/5.31 MiB
ruff ------------------------------ 4.40 MiB/11.02 MiB
duckdb ------------------------------ 4.00 MiB/19.27 MiB
playwright ------------------------------ 3.25 MiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠸ Preparing packages... (28/33)
pyright ------------------------------ 1.91 MiB/5.31 MiB
ruff ------------------------------ 5.69 MiB/11.02 MiB
duckdb ------------------------------ 5.30 MiB/19.27 MiB
playwright ------------------------------ 4.53 MiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠸ Preparing packages... (28/33)
pyright ------------------------------ 1.97 MiB/5.31 MiB
ruff ------------------------------ 7.11 MiB/11.02 MiB
duckdb ------------------------------ 6.72 MiB/19.27 MiB
playwright ------------------------------ 5.92 MiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠸ Preparing packages... (28/33)
pyright ------------------------------ 2.06 MiB/5.31 MiB
ruff ------------------------------ 8.55 MiB/11.02 MiB
duckdb ------------------------------ 8.15 MiB/19.27 MiB
playwright ------------------------------ 7.38 MiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠸ Preparing packages... (28/33)
pyright ------------------------------ 2.11 MiB/5.31 MiB
ruff ------------------------------ 9.94 MiB/11.02 MiB
duckdb ------------------------------ 9.59 MiB/19.27 MiB
playwright ------------------------------ 8.83 MiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠼ Preparing packages... (28/33)
pyright ------------------------------ 2.14 MiB/5.31 MiB
duckdb ------------------------------ 10.73 MiB/19.27 MiB
playwright ------------------------------ 9.98 MiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠼ Preparing packages... (28/33)
pyright ------------------------------ 2.14 MiB/5.31 MiB
duckdb ------------------------------ 11.01 MiB/19.27 MiB
playwright ------------------------------ 10.28 MiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠼ Preparing packages... (28/33)
pyright ------------------------------ 2.19 MiB/5.31 MiB
duckdb ------------------------------ 13.28 MiB/19.27 MiB
playwright ------------------------------ 12.55 MiB/43.05 MiB
Building scubaduck @ file:///workspace/scubaduck
⠼ Preparing packages... (28/33)
pyright ------------------------------ 2.22 MiB/5.31 MiB
duckdb ------------------------------ 15.44 MiB/19.27 MiB
playwright ------------------------------ 14.72 MiB/43.05 MiB
Built scubaduck @ file:///workspace/scubaduck
⠼ Preparing packages... (28/33)
pyright ------------------------------ 2.25 MiB/5.31 MiB
duckdb ------------------------------ 16.95 MiB/19.27 MiB
playwright ------------------------------ 16.19 MiB/43.05 MiB
⠼ Preparing packages... (28/33)
pyright ------------------------------ 2.25 MiB/5.31 MiB
duckdb ------------------------------ 17.42 MiB/19.27 MiB
playwright ------------------------------ 16.70 MiB/43.05 MiB
⠴ Preparing packages... (30/33)
pyright ------------------------------ 2.34 MiB/5.31 MiB
duckdb ------------------------------ 19.20 MiB/19.27 MiB
playwright ------------------------------ 18.92 MiB/43.05 MiB
⠴ Preparing packages... (30/33)
pyright ------------------------------ 2.37 MiB/5.31 MiB
playwright ------------------------------ 20.23 MiB/43.05 MiB
⠴ Preparing packages... (30/33)
pyright ------------------------------ 2.40 MiB/5.31 MiB
playwright ------------------------------ 22.98 MiB/43.05 MiB
⠴ Preparing packages... (30/33)
pyright ------------------------------ 2.48 MiB/5.31 MiB
playwright ------------------------------ 25.83 MiB/43.05 MiB
⠦ Preparing packages... (31/33)
pyright ------------------------------ 2.61 MiB/5.31 MiB
playwright ------------------------------ 27.76 MiB/43.05 MiB
⠦ Preparing packages... (31/33)
pyright ------------------------------ 2.73 MiB/5.31 MiB
playwright ------------------------------ 29.33 MiB/43.05 MiB
⠦ Preparing packages... (31/33)
pyright ------------------------------ 2.83 MiB/5.31 MiB
playwright ------------------------------ 31.31 MiB/43.05 MiB
⠦ Preparing packages... (31/33)
pyright ------------------------------ 2.94 MiB/5.31 MiB
playwright ------------------------------ 32.84 MiB/43.05 MiB
⠦ Preparing packages... (31/33)
pyright ------------------------------ 3.06 MiB/5.31 MiB
playwright ------------------------------ 34.84 MiB/43.05 MiB
⠧ Preparing packages... (31/33)
pyright ------------------------------ 3.20 MiB/5.31 MiB
playwright ------------------------------ 36.37 MiB/43.05 MiB
⠧ Preparing packages... (31/33)
pyright ------------------------------ 3.29 MiB/5.31 MiB
playwright ------------------------------ 38.02 MiB/43.05 MiB
⠧ Preparing packages... (31/33)
pyright ------------------------------ 3.45 MiB/5.31 MiB
playwright ------------------------------ 39.87 MiB/43.05 MiB
⠧ Preparing packages... (31/33)
pyright ------------------------------ 3.58 MiB/5.31 MiB
playwright ------------------------------ 41.11 MiB/43.05 MiB
⠇ Preparing packages... (31/33)
pyright ------------------------------ 3.78 MiB/5.31 MiB
playwright ------------------------------ 42.23 MiB/43.05 MiB
⠇ Preparing packages... (31/33)
pyright ------------------------------ 3.79 MiB/5.31 MiB
⠇ Preparing packages... (31/33)
pyright ------------------------------ 4.12 MiB/5.31 MiB
⠇ Preparing packages... (31/33)
pyright ------------------------------ 4.39 MiB/5.31 MiB
⠋ Preparing packages... (32/33)
pyright ------------------------------ 4.52 MiB/5.31 MiB
⠋ Preparing packages... (32/33)
Prepared 33 packages in 1.62s
░░░░░░░░░░░░░░░░░░░░ [0/0] Installing wheels...
░░░░░░░░░░░░░░░░░░░░ [0/33] Installing wheels...
░░░░░░░░░░░░░░░░░░░░ [0/33] pyee==13.0.0
░░░░░░░░░░░░░░░░░░░░ [1/33] pyee==13.0.0
░░░░░░░░░░░░░░░░░░░░ [1/33] pytest-base-url==2.1.0
█░░░░░░░░░░░░░░░░░░░ [2/33] pytest-base-url==2.1.0
█░░░░░░░░░░░░░░░░░░░ [2/33] pytest-playwright==0.7.0
█░░░░░░░░░░░░░░░░░░░ [3/33] pytest-playwright==0.7.0
█░░░░░░░░░░░░░░░░░░░ [3/33] python-slugify==8.0.4
██░░░░░░░░░░░░░░░░░░ [4/33] python-slugify==8.0.4
██░░░░░░░░░░░░░░░░░░ [4/33] six==1.17.0
███░░░░░░░░░░░░░░░░░ [5/33] six==1.17.0
███░░░░░░░░░░░░░░░░░ [5/33] iniconfig==2.1.0
███░░░░░░░░░░░░░░░░░ [6/33] iniconfig==2.1.0
███░░░░░░░░░░░░░░░░░ [6/33] pluggy==1.6.0
████░░░░░░░░░░░░░░░░ [7/33] pluggy==1.6.0
████░░░░░░░░░░░░░░░░ [7/33] idna==3.10
████░░░░░░░░░░░░░░░░ [8/33] idna==3.10
████░░░░░░░░░░░░░░░░ [8/33] execnet==2.1.1
█████░░░░░░░░░░░░░░░ [9/33] execnet==2.1.1
███████████████████░ [32/33] pyright==1.1.400
Installed 33 packages in 78ms
+ blinker==1.9.0
+ certifi==2025.4.26
+ charset-normalizer==3.4.2
+ click==8.2.0
+ duckdb==1.2.2
+ execnet==2.1.1
+ flask==3.1.1
+ greenlet==3.2.2
+ idna==3.10
+ iniconfig==2.1.0
+ itsdangerous==2.2.0
+ jinja2==3.1.6
+ markupsafe==3.0.2
+ nodeenv==1.9.1
+ packaging==25.0
+ playwright==1.52.0
+ pluggy==1.6.0
+ pyee==13.0.0
+ pyright==1.1.400
+ pytest==8.3.5
+ pytest-base-url==2.1.0
+ pytest-playwright==0.7.0
+ pytest-xdist==3.6.1
+ python-dateutil==2.9.0.post0
+ python-slugify==8.0.4
+ requests==2.32.3
+ ruff==0.11.10
+ scubaduck==0.1.0 (from file:///workspace/scubaduck)
+ six==1.17.0
+ text-unidecode==1.3
+ typing-extensions==4.13.2
+ urllib3==2.4.0
+ werkzeug==3.1.3
++ source .venv/bin/activate
+++ '[' -z '' ']'
+++ '[' -n x ']'
+++ SCRIPT_PATH=.venv/bin/activate
+++ '[' .venv/bin/activate = /tmp/8nIs79-setup_script.sh ']'
+++ deactivate nondestructive
+++ unset -f pydoc
+++ '[' -z '' ']'
+++ '[' -z '' ']'
+++ hash -r
+++ '[' -z '' ']'
+++ unset VIRTUAL_ENV
+++ unset VIRTUAL_ENV_PROMPT
+++ '[' '!' nondestructive = nondestructive ']'
+++ VIRTUAL_ENV=/workspace/scubaduck/.venv
+++ '[' linux-gnu = cygwin ']'
+++ '[' linux-gnu = msys ']'
+++ export VIRTUAL_ENV
+++ '[' -z '' ']'
+++ unset SCRIPT_PATH
+++ _OLD_VIRTUAL_PATH=/root/.cargo/bin:/root/.rbenv/shims:/root/.rbenv/bin:/root/.rbenv/shims:/root/.local/share/swiftly/bin:/root/.bun/bin:/root/.nvm/versions/node/v22.15.1/bin:/root/.pyenv/shims:3441PYENV_ROOT/shims:/root/.pyenv/bin:/usr/local/go/bin:/root/go/bin:/root/.rbenv/bin:/root/.rbenv/shims:/root/.bun/bin:/root/.local/bin:/root/.pyenv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+++ PATH=/workspace/scubaduck/.venv/bin:/root/.cargo/bin:/root/.rbenv/shims:/root/.rbenv/bin:/root/.rbenv/shims:/root/.local/share/swiftly/bin:/root/.bun/bin:/root/.nvm/versions/node/v22.15.1/bin:/root/.pyenv/shims:3441PYENV_ROOT/shims:/root/.pyenv/bin:/usr/local/go/bin:/root/go/bin:/root/.rbenv/bin:/root/.rbenv/shims:/root/.bun/bin:/root/.local/bin:/root/.pyenv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+++ export PATH
+++ '[' xscubaduck '!=' x ']'
+++ VIRTUAL_ENV_PROMPT='(scubaduck) '
+++ export VIRTUAL_ENV_PROMPT
+++ '[' -z '' ']'
+++ '[' -z '' ']'
+++ _OLD_VIRTUAL_PS1=
+++ PS1='(scubaduck) '
+++ export PS1
+++ alias pydoc
+++ true
+++ hash -r
++ python -c 'import os; import duckdb; con = duckdb.connect(); con.execute(f"SET http_proxy = '\''{os.getenv("HTTP_PROXY")}'\''"); con.execute("INSTALL '\''sqlite'\'';")'
++ playwright install chromium
Downloading Chromium 136.0.7103.25 (playwright build v1169) from https://cdn.playwright.dev/dbazure/download/playwright/builds/chromium/1169/chromium-linux.zip
167.7 MiB [] 0% 0.0s167.7 MiB [] 0% 27.7s167.7 MiB [] 0% 21.4s167.7 MiB [] 0% 14.2s167.7 MiB [] 0% 10.6s167.7 MiB [] 1% 5.9s167.7 MiB [] 2% 4.6s167.7 MiB [] 3% 3.6s167.7 MiB [] 5% 2.5s167.7 MiB [] 5% 2.4s167.7 MiB [] 6% 2.4s167.7 MiB [] 7% 2.2s167.7 MiB [] 9% 2.0s167.7 MiB [] 10% 1.9s167.7 MiB [] 12% 1.7s167.7 MiB [] 14% 1.6s167.7 MiB [] 15% 1.5s167.7 MiB [] 16% 1.5s167.7 MiB [] 18% 1.4s167.7 MiB [] 20% 1.3s167.7 MiB [] 21% 1.2s167.7 MiB [] 23% 1.2s167.7 MiB [] 25% 1.1s167.7 MiB [] 27% 1.1s167.7 MiB [] 29% 1.0s167.7 MiB [] 30% 1.0s167.7 MiB [] 32% 1.0s167.7 MiB [] 34% 0.9s167.7 MiB [] 37% 0.8s167.7 MiB [] 39% 0.8s167.7 MiB [] 40% 0.8s167.7 MiB [] 42% 0.7s167.7 MiB [] 44% 0.7s167.7 MiB [] 46% 0.7s167.7 MiB [] 48% 0.6s167.7 MiB [] 50% 0.6s167.7 MiB [] 52% 0.6s167.7 MiB [] 54% 0.5s167.7 MiB [] 56% 0.5s167.7 MiB [] 58% 0.5s167.7 MiB [] 60% 0.5s167.7 MiB [] 61% 0.5s167.7 MiB [] 63% 0.4s167.7 MiB [] 65% 0.4s167.7 MiB [] 67% 0.4s167.7 MiB [] 70% 0.3s167.7 MiB [] 71% 0.3s167.7 MiB [] 74% 0.3s167.7 MiB [] 76% 0.3s167.7 MiB [] 78% 0.2s167.7 MiB [] 80% 0.2s167.7 MiB [] 82% 0.2s167.7 MiB [] 84% 0.2s167.7 MiB [] 86% 0.1s167.7 MiB [] 88% 0.1s167.7 MiB [] 90% 0.1s167.7 MiB [] 93% 0.1s167.7 MiB [] 95% 0.0s167.7 MiB [] 97% 0.0s167.7 MiB [] 99% 0.0s167.7 MiB [] 100% 0.0s
Chromium 136.0.7103.25 (playwright build v1169) downloaded to /root/.cache/ms-playwright/chromium-1169
Downloading FFMPEG playwright build v1011 from https://cdn.playwright.dev/dbazure/download/playwright/builds/ffmpeg/1011/ffmpeg-linux.zip
2.3 MiB [] 0% 0.0s2.3 MiB [] 8% 0.2s2.3 MiB [] 24% 0.1s2.3 MiB [] 67% 0.0s2.3 MiB [] 100% 0.0s
FFMPEG playwright build v1011 downloaded to /root/.cache/ms-playwright/ffmpeg-1011
Downloading Chromium Headless Shell 136.0.7103.25 (playwright build v1169) from https://cdn.playwright.dev/dbazure/download/playwright/builds/chromium/1169/chromium-headless-shell-linux.zip
101.4 MiB [] 0% 0.0s101.4 MiB [] 0% 17.6s101.4 MiB [] 0% 10.1s101.4 MiB [] 0% 6.3s101.4 MiB [] 1% 3.8s101.4 MiB [] 4% 2.0s101.4 MiB [] 6% 1.6s101.4 MiB [] 9% 1.2s101.4 MiB [] 11% 1.1s101.4 MiB [] 13% 0.9s101.4 MiB [] 16% 0.8s101.4 MiB [] 20% 0.7s101.4 MiB [] 23% 0.7s101.4 MiB [] 26% 0.6s101.4 MiB [] 30% 0.5s101.4 MiB [] 35% 0.4s101.4 MiB [] 40% 0.4s101.4 MiB [] 43% 0.4s101.4 MiB [] 45% 0.4s101.4 MiB [] 48% 0.3s101.4 MiB [] 49% 0.3s101.4 MiB [] 51% 0.3s101.4 MiB [] 54% 0.3s101.4 MiB [] 57% 0.3s101.4 MiB [] 61% 0.2s101.4 MiB [] 65% 0.2s101.4 MiB [] 68% 0.2s101.4 MiB [] 72% 0.2s101.4 MiB [] 74% 0.2s101.4 MiB [] 77% 0.1s101.4 MiB [] 81% 0.1s101.4 MiB [] 86% 0.1s101.4 MiB [] 92% 0.0s101.4 MiB [] 98% 0.0s101.4 MiB [] 100% 0.0s
Chromium Headless Shell 136.0.7103.25 (playwright build v1169) downloaded to /root/.cache/ms-playwright/chromium_headless_shell-1169
++ echo 'source .venv/bin/activate'I'll check for AGENTS.md at the root of the repository as instructed.
bash: .venv/bin/activate: No such file or directory
root@3a541e64f0a7:/workspace# ls
scubaduckroot@3a541e64f0a7:/workspace# cd scubaduck
root@3a541e64f0a7:/workspace/scubaduck# ls
AGENTS.md codex_setup.sh scubaduck tests
README.md pyproject.toml stubs uv.lockroot@3a541e64f0a7:/workspace/scubaduck# cat AGENTS.md
- Use "source .venv/bin/activate" to activate the virtual environment before
running any commands
- ONLY use non-interactive commands like cat, sed, apply_patch to do edits.
Do NOT use interactive editors.
- Do NOT attempt to install packages. Only the packages specified in
pyproject.toml are available. You cannot add new packages. If you
desperately want another package, make a note of it in the final PR
description.
- Use conventional commits to format PR title
- There are no nested AGENTS.md files, this is the only agents file
- When using Playwright, ONLY use chromium browser. This browser is already
installed.
- Use "ruff check" to check lint, "ruff format" to autoformat files and
"pyright" to typecheck.
- When writing the PR description, include the original prompt supplied by the
user.
- When you add functionality to the server, add server tests to
tests/test_server.py. When you add features to the frontend, add frontend
tests to tests/test_web.pyI'll follow the AGENTS instructions, including activating the virtual environment and following PR formatting.
root@3a541e64f0a7:/workspace/scubaduck# cat codex_setup.sh
uv sync --frozen
source .venv/bin/activate
python -c "import os; import duckdb; con = duckdb.connect(); con.execute(f\"SET
http_proxy = '{os.getenv(\"HTTP_PROXY\")}'\"); con.execute(\"INSTALL 'sqlite';\"
)"
playwright install chromium
echo "source .venv/bin/activate" >> ~/.bashrcI think the sqlite extension for DuckDB is automatically installed. Let’s check the repo.
root@3a541e64f0a7:/workspace/scubaduck# ls scubaduck
__init__.py sample.csv server.py staticroot@3a541e64f0a7:/workspace/scubaduck# ls tests
conftest.py test_server.py test_web.pyI'll inspect the server code to make sure everything looks correct.
root@3a541e64f0a7:/workspace/scubaduck# sed -n '1,160p' scubaduck/server.py
from __future__ import annotations
from dataclasses import dataclass, field, replace
from typing import Any, Dict, List, Tuple, cast
import re
from datetime import datetime, timedelta, timezone
import time
from pathlib import Path
import os
import sqlite3
import traceback
import math
import duckdb
from dateutil import parser as dtparser
from flask import Flask, jsonify, request, send_from_directory
@dataclass
class Filter:
column: str
op: str
value: str | int | float | list[str] | None
@dataclass
class QueryParams:
start: str | None = None
end: str | None = None
order_by: str | None = None
order_dir: str = "ASC"
limit: int | None = None
columns: list[str] = field(default_factory=lambda: [])
filters: list[Filter] = field(default_factory=lambda: [])
derived_columns: dict[str, str] = field(default_factory=lambda: {})
graph_type: str = "samples"
group_by: list[str] = field(default_factory=lambda: [])
aggregate: str | None = None
show_hits: bool = False
x_axis: str | None = None
granularity: str = "Auto"
fill: str = "0"
table: str = "events"
time_column: str = "timestamp"
time_unit: str = "s"
def _normalize_sqlite_type(sql: str) -> str:
"""Map arbitrary SQLite column types to DuckDB-compatible types."""
t = sql.strip().upper()
if "(" in t:
t = t.split("(", 1)[0]
if "INT" in t:
# SQLite only has a single INTEGER type which is always 64-bit.
# Use DuckDB's BIGINT to avoid overflow when values exceed INT32.
return "BIGINT"
if any(key in t for key in ("CHAR", "CLOB", "TEXT")):
return "VARCHAR"
if "BLOB" in t:
return "BLOB"
if any(key in t for key in ("DOUBLE", "REAL", "FLOA", "NUMERIC", "DECIMAL"))
:
return "DOUBLE"
if "BOOL" in t:
return "BOOLEAN"
if "DATE" in t or "TIME" in t:
return "TIMESTAMP" if "TIME" in t else "DATE"
return "VARCHAR"
def _load_database(path: Path) -> duckdb.DuckDBPyConnection:
if not path.exists():
raise FileNotFoundError(path)
ext = path.suffix.lower()
if ext == ".csv":
con = duckdb.connect()
con.execute(
f"CREATE TABLE events AS SELECT * FROM read_csv_auto('{path.as_posix
()}')"
)
elif ext in {".db", ".sqlite"}:
con = duckdb.connect()
try:
con.execute("LOAD sqlite")
con.execute(f"ATTACH '{path.as_posix()}' AS db (TYPE SQLITE)")
tables = [
r[0]
for r in con.execute(
"SELECT name FROM db.sqlite_master WHERE type='table'"
).fetchall()
]
for t in tables:
con.execute(f'CREATE VIEW "{t}" AS SELECT * FROM db."{t}"')
except Exception:
sconn = sqlite3.connect(path)
tables = [
r[0]
for r in sconn.execute(
"SELECT name FROM sqlite_master WHERE type='table'"
).fetchall()
]
for t in tables:
info = sconn.execute(f'PRAGMA table_info("{t}")').fetchall()
col_defs = ", ".join(
f'"{r[1]}" {_normalize_sqlite_type(cast(str, r[2]))}' for r
in info
)
sql = f'CREATE TABLE "{t}" ({col_defs})'
try:
con.execute(sql)
except Exception:
print(f"Failed SQL: {sql}")
raise
placeholders = ",".join("?" for _ in info)
for row in sconn.execute(f'SELECT * FROM "{t}"'):
con.execute(f'INSERT INTO "{t}" VALUES ({placeholders})', ro
w)
sconn.close()
else:
con = duckdb.connect(path)
return con
_REL_RE = re.compile(
r"([+-]?\d+(?:\.\d*)?)\s*(hour|hours|day|days|week|weeks|fortnight|fortnight
s)",
re.IGNORECASE,
)
def parse_time(val: str | None) -> str | None:
"""Parse an absolute or relative time string into ``YYYY-MM-DD HH:MM:SS``.""
"
if val is None or val == "":
return None
s = val.strip()
if s.lower() == "now":
dt = datetime.now(timezone.utc)
return dt.replace(microsecond=0).strftime("%Y-%m-%d %H:%M:%S")
m = _REL_RE.fullmatch(s)
if m:
qty = float(m.group(1))
unit = m.group(2).lower()
delta: timedelta
if unit.startswith("hour"):
delta = timedelta(hours=qty)
elif unit.startswith("day"):
delta = timedelta(days=qty)
elif unit.startswith("week"):
delta = timedelta(weeks=qty)
elif unit.startswith("fortnight"):
delta = timedelta(weeks=2 * qty)
else: # pragma: no cover - defensive
raise ValueError(f"Unsupported unit: {unit}")
dt = datetime.now(timezone.utc) + delta
return dt.replace(microsecond=0).strftime("%Y-%m-%d %H:%M:%S")
dt = dtparser.parse(s)
return dt.replace(microsecond=0, tzinfo=None).strftime("%Y-%m-%d %H:%M:%S")
def _granularity_seconds(granularity: str, start: str | None, end: str | None) -
> int:root@3a541e64f0a7:/workspace/scubaduck# sed -n '160,320p' scubaduck/server.py
def _granularity_seconds(granularity: str, start: str | None, end: str | None) -
> int:
gran = granularity.lower()
mapping = {
"1 second": 1,
"5 seconds": 5,
"10 seconds": 10,
"30 seconds": 30,
"1 minute": 60,
"4 minutes": 240,
"5 minutes": 300,
"10 minutes": 600,
"15 minutes": 900,
"30 minutes": 1800,
"1 hour": 3600,
"3 hours": 10800,
"6 hours": 21600,
"1 day": 86400,
"1 week": 604800,
"30 days": 2592000,
}
if gran in mapping:
return mapping[gran]
if gran in {"auto", "fine"} and start and end:
try:
s = dtparser.parse(start)
e = dtparser.parse(end)
except Exception:
return 3600
total = max((e - s).total_seconds(), 1)
buckets = 100 if gran == "auto" else 500
return max(int(total // buckets), 1)
return 3600
def _time_expr(col: str, column_types: Dict[str, str] | None, unit: str) -> str:
"""Return SQL expression for column interpreted as timestamp."""
if column_types is None:
return col
ctype = column_types.get(col, "").upper()
if not any(t in ctype for t in ["TIMESTAMP", "DATE", "TIME"]):
if any(
t in ctype
for t in [
"INT",
"DECIMAL",
"REAL",
"DOUBLE",
"FLOAT",
"NUMERIC",
"HUGEINT",
]
):
divisor = {
"s": 1,
"ms": 1000,
"us": 1_000_000,
"ns": 1_000_000_000,
}.get(unit, 1)
if divisor == 1:
expr = f"CAST({col} AS BIGINT)"
else:
expr = f"CAST({col} / {divisor} AS BIGINT)"
return f"TIMESTAMP 'epoch' + INTERVAL '1 second' * {expr}"
return col
def build_query(params: QueryParams, column_types: Dict[str, str] | None = None)
-> str:
select_parts: list[str] = []
group_cols = params.group_by[:]
if params.graph_type == "timeseries":
sec = _granularity_seconds(params.granularity, params.start, params.end)
x_axis = params.x_axis or params.time_column
xexpr = _time_expr(x_axis, column_types, params.time_unit)
if params.start:
bucket_expr = (
f"TIMESTAMP '{params.start}' + INTERVAL '{sec} second' * "
f"CAST(floor((epoch({xexpr}) - epoch(TIMESTAMP '{params.start}')
)/{sec}) AS BIGINT)"
)
else:
bucket_expr = (
f"TIMESTAMP 'epoch' + INTERVAL '{sec} second' * "
f"CAST(floor(epoch({xexpr})/{sec}) AS BIGINT)"
)
select_parts.append(f"{bucket_expr} AS bucket")
group_cols = ["bucket"] + group_cols
has_agg = bool(group_cols) or params.aggregate is not None
if has_agg:
select_cols = (
group_cols[1:] if params.graph_type == "timeseries" else group_cols
)
select_parts.extend(select_cols)
agg = (params.aggregate or "avg").lower()
def agg_expr(col: str) -> str:
if agg.startswith("p"):
quant = float(agg[1:]) / 100
return f"quantile({col}, {quant})"
if agg == "count distinct":
return f"count(DISTINCT {col})"
if agg == "avg" and column_types is not None:
ctype = column_types.get(col, "").upper()
if "TIMESTAMP" in ctype or "DATE" in ctype or "TIME" in ctype:
return (
"TIMESTAMP 'epoch' + INTERVAL '1 second' * "
f"CAST(avg(epoch({col})) AS BIGINT)"
)
return f"{agg}({col})"
for col in params.columns:
if col in group_cols:
continue
select_parts.append(f"{agg_expr(col)} AS {col}")
if params.show_hits:
select_parts.insert(len(group_cols), "count(*) AS Hits")
else:
select_parts.extend(params.columns)
if has_agg and params.derived_columns:
inner_params = replace(
params,
derived_columns={},
order_by=None,
limit=None,
)
inner_sql = build_query(inner_params, column_types)
outer_select = ["t.*"] + [
f"{expr} AS {name}" for name, expr in params.derived_columns.items()
]
indented_inner = "\n".join(" " + line for line in inner_sql.splitline
s())
lines = [
f"SELECT {', '.join(outer_select)}",
"FROM (",
indented_inner,
") t",
]
if params.order_by:
lines.append(f"ORDER BY {params.order_by} {params.order_dir}")
elif params.graph_type == "timeseries":
lines.append("ORDER BY bucket")
if params.limit is not None:
lines.append(f"LIMIT {params.limit}")
return "\n".join(lines)
for name, expr in params.derived_columns.items():
select_parts.append(f"{expr} AS {name}")
select_clause = ", ".join(select_parts) if select_parts else "*"
lines = [f"SELECT {select_clause}", f'FROM "{params.table}"']
where_parts: list[str] = []
time_expr = _time_expr(params.time_column, column_types, params.time_unit)
if params.start:
where_parts.append(f"{time_expr} >= '{params.start}'")
if params.end:
where_parts.append(f"{time_expr} <= '{params.end}'")
for f in params.filters:
op = f.op
if op in {"empty", "!empty"}:
val = "''"
else:
if f.value is None:
continue
if isinstance(f.value, list):I'll need to remove the fallback path from _load_database and investigate the sqlite_master error.
root@3a541e64f0a7:/workspace/scubaduck# sed -n '1,200p' tests/test_server.py
from __future__ import annotations
import json
from pathlib import Path
from typing import cast
import duckdb
from scubaduck import server
import pytest
def test_basic_query() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-02 00:00:00",
"order_by": "timestamp",
"order_dir": "ASC",
"limit": 10,
"columns": ["timestamp", "event", "value", "user"],
"filters": [],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert data
rows = data["rows"]
# We expect first three rows (until 2024-01-02 00:00:00)
assert len(rows) == 3
assert rows[0][1] == "login"
assert rows[1][1] == "logout"
def test_js_served() -> None:
app = server.app
client = app.test_client()
rv = client.get("/js/chip_input.js")
assert rv.status_code == 200
assert b"initChipInput" in rv.data
def test_filter_multi_token() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-02 03:00:00",
"order_by": "timestamp",
"limit": 10,
"columns": ["timestamp", "event", "value", "user"],
"filters": [{"column": "user", "op": "=", "value": ["alice", "charlie"]}
],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert data
rows = data["rows"]
# Should only return rows for alice and charlie
assert len(rows) == 3
assert rows[0][3] == "alice"
assert rows[-1][3] == "charlie"
def test_empty_filter_is_noop() -> None:
app = server.app
client = app.test_client()
base_payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"limit": 100,
"columns": ["timestamp", "event", "value", "user"],
}
no_filter = {**base_payload, "filters": []}
empty_filter = {
**base_payload,
"filters": [{"column": "user", "op": "=", "value": None}],
}
rv1 = client.post(
"/api/query", data=json.dumps(no_filter), content_type="application/json
"
)
rv2 = client.post(
"/api/query", data=json.dumps(empty_filter), content_type="application/j
son"
)
rows1 = rv1.get_json()["rows"]
rows2 = rv2.get_json()["rows"]
assert rows1 == rows2
def test_select_columns() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"order_by": "timestamp",
"limit": 10,
"columns": ["timestamp", "user"],
"filters": [],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert data
rows = data["rows"]
assert len(rows[0]) == 2
assert rows[0][1] == "alice"
def test_string_filter_ops() -> None:
app = server.app
client = app.test_client()
base = {
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"order_by": "timestamp",
"limit": 100,
"columns": ["timestamp", "event", "value", "user"],
}
contains = {
**base,
"filters": [{"column": "user", "op": "contains", "value": "ali"}],
}
rv = client.post(
"/api/query", data=json.dumps(contains), content_type="application/json"
)
rows = rv.get_json()["rows"]
assert all("ali" in r[3] for r in rows)
regex = {
**base,
"filters": [{"column": "user", "op": "~", "value": "^a.*"}],
}
rv = client.post(
"/api/query", data=json.dumps(regex), content_type="application/json"
)
rows = rv.get_json()["rows"]
assert all(r[3].startswith("a") for r in rows)
assert len(rows) == 2
not_empty = {**base, "filters": [{"column": "user", "op": "!empty"}]}
rv = client.post(
"/api/query", data=json.dumps(not_empty), content_type="application/json
"
)
assert len(rv.get_json()["rows"]) == 4
def _make_payload() -> dict[str, object]:
return {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-02 00:00:00",
"order_by": "timestamp",
"order_dir": "ASC",
"limit": 10,
"columns": ["timestamp", "event", "value", "user"],
"filters": [],
}
def test_database_types(tmp_path: Path) -> None:
csv_file = tmp_path / "events.csv"
csv_file.write_text(Path("scubaduck/sample.csv").read_text())
sqlite_file = tmp_path / "events.sqlite"
import sqlite3
conn = sqlite3.connect(sqlite_file)
conn.execute(
"CREATE TABLE events (timestamp TEXT, event TEXT, value INTEGER, user TE
XT)"
)
with open(csv_file) as f:
next(f)
for line in f:
ts, ev, val, user = line.strip().split(",")
conn.execute(
"INSERT INTO events VALUES (?, ?, ?, ?)", (ts, ev, int(val), use
r)
)
conn.commit()
conn.close() # pyright: ignore[reportUnknownMemberType, reportAttributeAcce
ssIssue]
duckdb_file = tmp_path / "events.duckdb"
con = duckdb.connect(duckdb_file)
con.execute(
f"CREATE TABLE events AS SELECT * FROM read_csv_auto('{csv_file.as_posix
()}')"
)
con.close() # pyright: ignore[reportUnknownMemberType, reportAttributeAcces
sIssue]
for db in (csv_file, sqlite_file, duckdb_file):
app = server.create_app(db)
client = app.test_client()root@3a541e64f0a7:/workspace/scubaduck# sed -n '200,400p' tests/test_server.py
client = app.test_client()
payload = _make_payload()
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/js
on"
)
rows = rv.get_json()["rows"]
assert len(rows) == 3
def test_sqlite_longvarchar(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) ->
None:
sqlite_file = tmp_path / "events.sqlite"
import sqlite3
conn = sqlite3.connect(sqlite_file)
conn.execute(
"CREATE TABLE events (timestamp TEXT, url LONGVARCHAR, title VARCHAR(10)
)"
)
conn.execute(
"INSERT INTO events VALUES ('2024-01-01 00:00:00', 'https://a.com', 'Hom
e')"
)
conn.commit()
conn.close() # pyright: ignore[reportUnknownMemberType, reportAttributeAcce
ssIssue]
from typing import Any
real_connect = duckdb.connect
def failing_connect(*args: Any, **kwargs: Any) -> Any:
real = real_connect(*args, **kwargs)
class Wrapper:
def __init__(self, con: duckdb.DuckDBPyConnection) -> None:
self.con = con
self._failed = False
def execute(self, sql: str, *a: Any, **kw: Any):
if not self._failed and sql == "LOAD sqlite":
self._failed = True
raise RuntimeError("fail")
return self.con.execute(sql, *a, **kw)
def __getattr__(self, name: str) -> object:
return getattr(self.con, name)
return Wrapper(real)
monkeypatch.setattr(server.duckdb, "connect", failing_connect)
app = server.create_app(sqlite_file)
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-01 01:00:00",
"order_by": "timestamp",
"columns": ["timestamp", "url", "title"],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert data["rows"][0][1] == "https://a.com"
def test_sqlite_bigint(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
sqlite_file = tmp_path / "big.sqlite"
import sqlite3
conn = sqlite3.connect(sqlite_file)
conn.execute("CREATE TABLE events (timestamp TEXT, value INTEGER)")
big_value = 13385262862605259
conn.execute(
"INSERT INTO events VALUES ('2024-01-01 00:00:00', ?)",
(big_value,),
)
conn.commit()
conn.close() # pyright: ignore[reportUnknownMemberType, reportAttributeAcce
ssIssue]
from typing import Any
real_connect = duckdb.connect
def failing_connect(*args: Any, **kwargs: Any) -> Any:
real = real_connect(*args, **kwargs)
class Wrapper:
def __init__(self, con: duckdb.DuckDBPyConnection) -> None:
self.con = con
self._failed = False
def execute(self, sql: str, *a: Any, **kw: Any):
if not self._failed and sql == "LOAD sqlite":
self._failed = True
raise RuntimeError("fail")
return self.con.execute(sql, *a, **kw)
def __getattr__(self, name: str) -> object:
return getattr(self.con, name)
return Wrapper(real)
monkeypatch.setattr(server.duckdb, "connect", failing_connect)
app = server.create_app(sqlite_file)
client = app.test_client()
payload = {
"table": "events",
"order_by": "timestamp",
"columns": ["timestamp", "value"],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert data["rows"][0][1] == big_value
def test_integer_time_column(tmp_path: Path) -> None:
csv_file = tmp_path / "events.csv"
csv_file.write_text("created,event\n1704067200,login\n1704070800,logout\n")
app = server.create_app(csv_file)
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-01 01:00:00",
"order_by": "created",
"columns": ["created", "event"],
"time_column": "created",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert len(data["rows"]) == 2
def test_integer_time_unit_ms(tmp_path: Path) -> None:
csv_file = tmp_path / "events.csv"
csv_file.write_text("created,event\n1704067200000,login\n1704070800000,logou
t\n")
app = server.create_app(csv_file)
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-01 01:00:00",
"order_by": "created",
"columns": ["created", "event"],
"time_column": "created",
"time_unit": "ms",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert len(data["rows"]) == 2
def test_integer_time_unit_us_default_start_end(tmp_path: Path) -> None:
csv_file = tmp_path / "events.csv"
csv_file.write_text(
"created,event\n1704067200000000,login\n1704070800000000,logout\n"
)
app = server.create_app(csv_file)
client = app.test_client()
payload = {
"table": "events",
"order_by": "created",
"columns": ["created", "event"],
"time_column": "created",
"time_unit": "us",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert data["start"] == "2024-01-01 00:00:00"
assert data["end"] == "2024-01-01 01:00:00"
assert len(data["rows"]) == 2
def test_envvar_db(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
csv_file = tmp_path / "custom.csv"
csv_file.write_text("timestamp,event,value,user\n2024-01-01 00:00:00,login,5
,bob\n")
monkeypatch.setenv("SCUBADUCK_DB", str(csv_file))
app = server.create_app()
client = app.test_client()
payload = _make_payload()
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
rows = rv.get_json()["rows"]
assert len(rows) == 1
def test_envvar_db_missing(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> N
one:with pytest.raises(FileNotFoundError):
server.create_app()
def test_group_by_table() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "table",
"order_by": "user",
"limit": 10,
"columns": ["value"],
"group_by": ["user"],
"aggregate": "Sum",
"show_hits": True,
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
rows = rv.get_json()["rows"]
assert rows[0][0] == "alice"
assert rows[0][1] == 2
assert rows[0][2] == 40
def test_relative_time_query(monkeypatch: pytest.MonkeyPatch) -> None:
app = server.app
client = app.test_client()
from datetime import datetime
fixed_now = datetime(2024, 1, 2, 4, 0, 0)
class FixedDateTime(datetime):
@classmethod
def now(cls, tz=None): # type: ignore[override]
return fixed_now if tz is None else fixed_now.astimezone(tz)
monkeypatch.setattr(server, "datetime", FixedDateTime)
payload = {
"table": "events",
"start": "-1 hour",
"end": "now",
"order_by": "timestamp",
"limit": 100,
"columns": ["timestamp", "event", "value", "user"],
"filters": [],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert len(data["rows"]) == 1
assert data["rows"][0][3] == "charlie"
def test_invalid_time_error() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "nonsense",
"end": "now",
"order_by": "timestamp",
"limit": 10,
"columns": ["timestamp"],
"filters": [],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 400
assert "error" in data
def test_query_error_returns_sql_and_traceback() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"columns": ["event"],
"group_by": ["user"],
"aggregate": "avg",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 400
assert "error" in data
def test_table_unknown_column_error() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "table",
"order_by": "timestamp",
"limit": 100,
"columns": ["user", "Hits", "value"],
"group_by": ["user"],
"aggregate": "Count",
"show_hits": True,
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 400
assert "Unknown column" in data["error"]
def test_samples_view_rejects_group_by() -> None:
...
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert len(data["rows"]) == 2
assert all(r[1] == "alice" for r in data["rows"])
def test_timeseries_auto_and_fine_buckets() -> None:
app = server.app
client = app.test_client()
def run(gran: str) -> None:
payload = {
"start": "2024-01-01 00:00:00",
"end": "2024-01-02 03:00:00",
"graph_type": "timeseries",
"columns": ["value"],
"x_axis": "timestamp",
"granularity": gran,
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/js
on"
)
data = rv.get_json()
assert rv.status_code == 200
from dateutil import parser
start = parser.parse(cast(str, payload["start"])).replace(tzinfo=None)
buckets = [
parser.parse(cast(str, r[0])).replace(tzinfo=None) for r in data["ro
ws"]
]
assert buckets[0] == start
if len(buckets) > 1:
step = (buckets[1] - buckets[0]).total_seconds()
assert step % data["bucket_size"] == 0
assert any(r[1] != 0 for r in data["rows"])
run("Auto")
run("Fine")
def test_timeseries_string_column_error() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "timeseries",
"limit": 7,
"columns": ["timestamp", "event", "value", "user"],
"x_axis": "timestamp",
"granularity": "1 hour",
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 400
assert "Aggregate" in data["error"]
def test_derived_column_basic() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"limit": 10,
"columns": ["timestamp"],
"derived_columns": {"val2": "value * 2"},
"filters": [],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert data["rows"][0][1] == 20
def test_timeseries_derived_column() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-03 00:00:00",
"graph_type": "timeseries",
"granularity": "1 hour",
"limit": 7,
"columns": ["value"],
"derived_columns": {"derived_1": "value * 2"},
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
rows = data["rows"]
assert all(r[2] == r[1] * 2 for r in rows)
def test_default_start_end_returned() -> None:
app = server.app
client = app.test_client()
payload = {
"table": "events",
"order_by": "timestamp",
"limit": 5,
"columns": ["timestamp"],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert data["start"] == "2024-01-01 00:00:00"
assert data["end"] == "2024-01-02 03:00:00"The tests are fine for now. Next, I'll look beyond _load_database in server.py.
root@3a541e64f0a7:/workspace/scubaduck# grep -n "_load_database" -n scubaduck/se
rver.py
72:def _load_database(path: Path) -> duckdb.DuckDBPyConnection:
359: con = _load_database(db_path)Heading to line 330 to check the changes!
if group_cols:
lines.append("GROUP BY " + ", ".join(group_cols))
if params.order_by:
lines.append(f"ORDER BY {params.order_by} {params.order_dir}")
elif params.graph_type == "timeseries":
lines.append("ORDER BY bucket")
if params.limit is not None:
lines.append(f"LIMIT {params.limit}")
return "\n".join(lines)
def create_app(db_file: str | Path | None = None) -> Flask:
app = Flask(__name__, static_folder="static")
if db_file is None:
env_db = os.environ.get("SCUBADUCK_DB")
if env_db:
db_file = env_db
db_path = Path(db_file or Path(__file__).with_name("sample.csv")).resolve()
con = _load_database(db_path)
tables = [r[0] for r in con.execute("SHOW TABLES").fetchall()]
if not tables:
raise ValueError("No tables found in database")
default_table = tables[0]
columns_cache: Dict[str, Dict[str, str]] = {}
def get_columns(table: str) -> Dict[str, str]:
if table not in columns_cache:
rows = con.execute(f'PRAGMA table_info("{table}")').fetchall()
if not rows:
raise ValueError(f"Unknown table: {table}")
columns_cache[table] = {r[1]: r[2] for r in rows}
return columns_cache[table]
sample_cache: Dict[Tuple[str, str, str], Tuple[List[str], float]] = {}
CACHE_TTL = 60.0
CACHE_LIMIT = 200
@app.route("/")
def index() -> Any: # pyright: ignore[reportUnusedFunction]
assert app.static_folder is not Noneroot@3a541e64f0a7:/workspace/scubaduck# sed -n '370,420p' scubaduck/server.py
raise ValueError(f"Unknown table: {table}")
columns_cache[table] = {r[1]: r[2] for r in rows}
return columns_cache[table]
sample_cache: Dict[Tuple[str, str, str], Tuple[List[str], float]] = {}
CACHE_TTL = 60.0
CACHE_LIMIT = 200
@app.route("/")
def index() -> Any: # pyright: ignore[reportUnusedFunction]
assert app.static_folder is not None
return send_from_directory(app.static_folder, "index.html")
@app.route("/js/<path:filename>")
def js(filename: str) -> Any: # pyright: ignore[reportUnusedFunction]
assert app.static_folder is not None
folder = Path(app.static_folder) / "js"
return send_from_directory(folder, filename)
@app.route("/api/tables")
def tables_endpoint() -> Any: # pyright: ignore[reportUnusedFunction]
return jsonify(tables)
@app.route("/api/columns")
def columns() -> Any: # pyright: ignore[reportUnusedFunction]
table = request.args.get("table", default_table)
rows = con.execute(f'PRAGMA table_info("{table}")').fetchall()
return jsonify([{"name": r[1], "type": r[2]} for r in rows])
def _cache_get(key: Tuple[str, str, str]) -> List[str] | None:
item = sample_cache.get(key)
if item is None:
return None
vals, ts = item
if time.time() - ts > CACHE_TTL:
del sample_cache[key]
return None
sample_cache[key] = (vals, time.time())
return vals
def _cache_set(key: Tuple[str, str, str], vals: List[str]) -> None:
sample_cache[key] = (vals, time.time())
if len(sample_cache) > CACHE_LIMIT:
oldest = min(sample_cache.items(), key=lambda kv: kv[1][1])[0]
del sample_cache[oldest]
@app.route("/api/samples")
def sample_values() -> Any: # pyright: ignore[reportUnusedFunction]
table = request.args.get("table", default_table)
column = request.args.get("column")
substr = request.args.get("q", "")root@3a541e64f0a7:/workspace/scubaduck# sed -n '420,800p' scubaduck/server.py
substr = request.args.get("q", "")
column_types = get_columns(table)
if not column or column not in column_types:
return jsonify([])
ctype = column_types[column].upper()
if "CHAR" not in ctype and "STRING" not in ctype and "VARCHAR" not in ct
ype:
return jsonify([])
key = (table, column, substr)
cached = _cache_get(key)
if cached is not None:
return jsonify(cached)
rows = con.execute(
f"SELECT DISTINCT {column} FROM \"{table}\" WHERE CAST({column} AS V
ARCHAR) ILIKE '%' || ? || '%' LIMIT 20",
[substr],
).fetchall()
values = [r[0] for r in rows]
_cache_set(key, values)
return jsonify(values)
@app.route("/api/query", methods=["POST"])
def query() -> Any: # pyright: ignore[reportUnusedFunction]
payload = request.get_json(force=True)
try:
start = parse_time(payload.get("start"))
end = parse_time(payload.get("end"))
except Exception as exc:
return jsonify({"error": str(exc)}), 400
params = QueryParams(
start=start,
end=end,
order_by=payload.get("order_by"),
order_dir=payload.get("order_dir", "ASC"),
limit=payload.get("limit"),
columns=payload.get("columns", []),
derived_columns=payload.get("derived_columns", {}),
graph_type=payload.get("graph_type", "samples"),
group_by=payload.get("group_by", []),
aggregate=payload.get("aggregate"),
show_hits=payload.get("show_hits", False),
x_axis=payload.get("x_axis"),
granularity=payload.get("granularity", "Auto"),
fill=payload.get("fill", "0"),
table=payload.get("table", default_table),
time_column=payload.get("time_column", "timestamp"),
time_unit=payload.get("time_unit", "s"),
)
for f in payload.get("filters", []):
params.filters.append(Filter(f["column"], f["op"], f.get("value")))
if params.table not in tables:
return jsonify({"error": "Invalid table"}), 400
column_types = get_columns(params.table)
if params.time_column not in column_types:
return jsonify({"error": "Invalid time_column"}), 400
if params.time_unit not in {"s", "ms", "us", "ns"}:
return jsonify({"error": "Invalid time_unit"}), 400
if params.graph_type not in {"table", "timeseries"} and (
params.group_by or params.aggregate or params.show_hits
):
return (
jsonify(
{
"error": "group_by, aggregate and show_hits are only val
id for table or timeseries view"
}
),
400,
)
valid_cols = set(column_types.keys())
valid_cols.update(params.derived_columns.keys())
if params.graph_type == "timeseries":
if params.x_axis is None:
params.x_axis = params.time_column
if params.x_axis not in valid_cols:
return jsonify({"error": "Invalid x_axis"}), 400
ctype = column_types.get(params.x_axis, "").upper()
is_time = any(t in ctype for t in ["TIMESTAMP", "DATE", "TIME"])
is_numeric = any(
t in ctype
for t in [
"INT",
"DECIMAL",
"REAL",
"DOUBLE",
"FLOAT",
"NUMERIC",
"HUGEINT",
]
)
if not (is_time or is_numeric):
return jsonify({"error": "x_axis must be a time column"}), 400
for col in params.columns:
if col not in valid_cols:
return jsonify({"error": f"Unknown column: {col}"}), 400
for col in params.group_by:
if col not in valid_cols:
return jsonify({"error": f"Unknown column: {col}"}), 400
if params.order_by and params.order_by not in valid_cols:
return jsonify({"error": f"Unknown column: {params.order_by}"}), 400
if params.group_by or params.graph_type == "timeseries":
agg = (params.aggregate or "avg").lower()
if agg.startswith("p") or agg == "sum":
need_numeric = True
allow_time = False
elif agg == "avg" or agg in {"min", "max"}:
need_numeric = False
allow_time = True
else:
need_numeric = False
allow_time = False
if need_numeric or allow_time:
for c in params.columns:
if c in params.group_by or c == params.x_axis:
continue
...
to column {c}",
}
),
400,
)
if allow_time and not (is_numeric or is_time):
return (
jsonify(
{
"error": f"Aggregate {agg} cannot be applied
to column {c}",
}
),
400,
)
if params.start is None or params.end is None:
axis = params.x_axis or params.time_column
row = cast(
tuple[datetime | None, datetime | None],
con.execute(
f'SELECT min({axis}), max({axis}) FROM "{params.table}"'
).fetchall()[0],
)
mn, mx = row
divisor = {
"s": 1,
"ms": 1000,
"us": 1_000_000,
"ns": 1_000_000_000,
}.get(params.time_unit, 1)
if isinstance(mn, (int, float)):
try:
mn = datetime.fromtimestamp(int(mn) / divisor, tz=timezone.u
tc)
except Exception:
return (
jsonify(
{
"error": (
f"Invalid time value {mn} for column {axis}"
f" with time_unit {params.time_unit}"
)
}
),
400,
)
if isinstance(mx, (int, float)):
try:
mx = datetime.fromtimestamp(int(mx) / divisor, tz=timezone.u
tc)
except Exception:
return (
jsonify(
{
"error": (
f"Invalid time value {mx} for column {axis}"
f" with time_unit {params.time_unit}"
)
}
),
400,
)
if params.start is None and mn is not None:
params.start = (
mn.strftime("%Y-%m-%d %H:%M:%S") if not isinstance(mn, str)
else mn
)
if params.end is None and mx is not None:
params.end = (
mx.strftime("%Y-%m-%d %H:%M:%S") if not isinstance(mx, str)
else mx
)
bucket_size: int | None = None
if params.graph_type == "timeseries":
bucket_size = _granularity_seconds(
params.granularity,
params.start if isinstance(params.start, str) else None,
params.end if isinstance(params.end, str) else None,
)
if (
params.limit is not None
and params.start is not None
and params.end is not None
):
try:
start_dt = dtparser.parse(params.start)
end_dt = dtparser.parse(params.end)
buckets = math.ceil(
(end_dt - start_dt).total_seconds() / bucket_size
)
if buckets > 1:
params.limit *= buckets
except Exception:
pass
sql = build_query(params, column_types)
try:
rows = con.execute(sql).fetchall()
except Exception as exc:
tb = traceback.format_exc()
print(f"Query failed:\n{sql}\n{tb}")
return (
jsonify({"sql": sql, "error": str(exc), "traceback": tb}),
400,
)
result: Dict[str, Any] = {"sql": sql, "rows": rows}
if params.start is not None:
result["start"] = str(params.start)
if params.end is not None:
result["end"] = str(params.end)
if bucket_size is not None:
result["bucket_size"] = bucket_size
return jsonify(result)
return app
app = create_app()
if __name__ == "__main__":
app.run(debug=True)I'll take a look at the README for documentation on this.
, real-time database part of Scuba is quite difficult (and expensive) to replica
te, but I also really like Scuba's UI for doing queries, and I have found myself
wishing that I have access to it even for "small" databases, e.g., I have a sql
ite dataset I want to explore.
Pivotal ideas:
* Time series by default. In the dedicated "time series" view, there are many fe
atures specifically oriented towards working towards tables that represent event
s that occurred over time: the start, end, compare, aggregate and granularity fi
elds all specially privilege the timestamp field. In fact, you can't log events
to Scuba's backing data store without a timestamp, they always come with one. (S
cuba also supports other views that don't presuppose a time series, but the time
series is the most beloved and well used view.) This is in contrast to typical
software which tries to generalize to arbitrary data first, with time series bei
ng added on later.
* It's all about exploration. Scuba is predicated on the idea that you don't kno
w what you're looking for, that you are going to spend time tweaking queries and
changing filters/grouping as part of an investigation to figure out why a syste
m behaves the way it is. So the filters/comparisons/groupings you want to edit a
re always visible on the left sidebar, with the expectation that you're going to
tweak the query to look at something else. Similarly, all the parameters of you
r query get saved into your URL, so your browser history can double up as a quer
y history / you can easily share a query with someone else. This is contrast to
typical software which is often oriented to making pretty dashboards and reports
. (This function is important too, but it's not what I want in exploration mode!
)
* You can fix data problems in the query editor. It's pretty common to have mess
ed up and ended up with a database that doesn't have exactly the columns you nee
d, or some columns that are corrupted in some way. Scuba has pretty robust suppo
rt for defining custom columns with arbitrary SQL functions, grouping over them
as if they were native functions, and doing so with minimal runtime cost (Scuba
aims to turn around your query in milliseconds!) Having to go and run a huge dat
a pipeline to fix your data is a big impediment to exploration; quick and easy c
ustom columns means you can patch over problems when you're investigating and fi
x them for real later.
We're going to build a exploratory data analysis tool like Scuba for time series
database (i.e., a database with a mandatory timestamp representing the time an
event occurred). We'll use DuckDB as the underlying SQL engine served from a Py
thon server, and render the GUI/results as a webpage with vanilla HTML and JS. W
e'll use choices.js to support token inputs. We define a token input to mean a
text input element where as you type a dropdown displays with valid values, and
if you select one or press enter, the selection turns into a token/chip that can
only be deleted as one unit.
To start, we are going to support one views: samples. The samples view only all
ows you to view individual samples from the database, subject to a filter. Our m
ain UI concept is that there is a left sidebar that is the query editor, and the
right side that shows the view. The sidebar is always visible and defaults to
the query parameters of the current view. After you make changes to the query,
clicking the "Dive" button updates the view. The URL of the page encodes all of
the values of the query (and gets updated when you Dive), so the browser's back
button lets you view previous queries.
The query editor's job is to generate a SQL query, which then is applied on the
database, and then the result visualized according to the view.
Here are the settings you can apply to the query. The help text should show up w
hen you mouse over the field name:
* Start/End - Help text: "Sets the start/end of the time range to query. Can be
any kind of datetime string. For example: 'April 23, 2014' or 'yesterday'." The
UI for this selector supports both relative selections (now, -1 hour, -3 hours,
-12 hours, -1 day, -3 days, -1 week, -1 fortnight, -30 days, -90 days) as well a
s specifying an absolute date. The way this field is rendered is there is a fre
e form text box, a drop down arrow (for the relative selectors), and then a cale
ndar button (for date selection).
* Order By - Help text: "Choose a column to sort results by." There is an ASC/D
ESC toggle next to it.
* Limit - Help text: "Choose the maximum number of results to show in the chart
after any aggregations have been applied. For example, a limit of 10 will show
no more than 10 rows for a table, etc."
* Filters - You can create as many filters as you want. You can either write a f
ilter using a UI or manual SQL. In the UI, filter consists of a column name, a r
elation (e.g., =, !=, <, >) and then a text field. The text field is a token inp
ut. It accepts multiple tokens for = relation, in which case we match using an O
R for all options.
There is also a "Columns" tab which lets you view all fields in the table, organ
ized by their type. You can also define derived columns, by specifying a column
name and SQL expression. Derived columns can be used for all parts of the UI, in
cluding filters/group by/etc. Columns have checkboxes indicating if we should SE
LECT them or not. Each selected column shows up in the graph. There is an All/N
one link which can be used to select/deselect all checkboxes.
The query UI constructs a SQL query that intuitively has this form:
```
SELECT column, column, ...,
FROM table
WHERE time >= min-timestamp
AND time <= max-timestamp
[AND condition ...]
ORDER BY aggregate(column)
LIMIT number
```
You should write tests for the server backend, demonstrating that at specific qu
ery values we get back the correct rows of data.
## Running the serverThe issue might be that the schema is main, not db. I'll check this.
root@3a541e64f0a7:/workspace/scubaduck# source .venv/bin/activate
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck# python - <<'PY'
> import duckdb, tempfile, sqlite3
> import os, pathlib
> import textwrap
> with tempfile.TemporaryDirectory() as td:
> path = pathlib.Path(td) / 'test.sqlite'
> conn = sqlite3.connect(path)
> conn.execute('CREATE TABLE t(a int)')
> conn.commit()
> conn.close()
>
> con = duckdb.connect()
> con.execute('LOAD sqlite')
> con.execute(f"ATTACH '{path}' AS db (TYPE SQLITE)")
> print(con.execute("SHOW TABLES").fetchall())
> print(con.execute("SELECT name FROM sqlite_master WHERE type='table'").fet
chall())
> print(con.execute("SELECT name FROM db.sqlite_master WHERE type='table'").
fetchall())
> print(con.execute("SELECT name FROM main.sqlite_master WHERE type='table'"
).fetchall())
> PY
[]
[('t',)]
Traceback (most recent call last):
File "<stdin>", line 16, in <module>
duckdb.duckdb.CatalogException: Catalog Error: Table with name sqlite_master doe
s not exist!
Did you mean "main.sqlite_master or memory.sqlite_master"?
LINE 1: SELECT name FROM db.sqlite_master WHERE type='table'
^
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck#It looks like the alias "db" refers to the attached SQLite database, not the schema.
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck# python - <<'PY'
> import duckdb, sqlite3, pathlib, tempfile
>
> with tempfile.TemporaryDirectory() as td:
> path = pathlib.Path(td) / 'test.sqlite'
> conn = sqlite3.connect(path)
> conn.execute('CREATE TABLE t(a int)')
> conn.commit(); conn.close()
>
> con = duckdb.connect()
> con.execute('LOAD sqlite')
> con.execute(f"ATTACH '{path}' AS db (TYPE SQLITE)")
> # show schemas
> print('SHOW SCHEMAS:', con.execute('SHOW SCHEMAS').fetchall())
> print('SHOW TABLES db:', con.execute('SHOW TABLES FROM db').fetchall())
> print('SHOW TABLES main:', con.execute('SHOW TABLES FROM main').fetchall()
)
> print('SHOW TABLES memory:', con.execute('SHOW TABLES FROM memory').fetcha
ll())
> PYTraceback (most recent call last):
File "<stdin>", line 13, in <module>
duckdb.duckdb.CatalogException: Catalog Error: Table with name SCHEMAS does not
exist!
LINE 1: SHOW SCHEMAS
^
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck#This DuckDB version might not support "SHOW SCHEMAS," so I'll try "pragma show_tables."
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck# python - <<'PY'
> import duckdb, sqlite3, pathlib, tempfile
>
> with tempfile.TemporaryDirectory() as td:
> path = pathlib.Path(td) / 'test.sqlite'
> conn = sqlite3.connect(path)
> conn.execute('CREATE TABLE t(a int)')
> conn.commit(); conn.close()
>
> con = duckdb.connect()
> con.execute('LOAD sqlite')
> con.execute(f"ATTACH '{path}' AS db (TYPE SQLITE)")
> print('PRAGMA show_tables main:', con.execute("SELECT * FROM main.sqlite_m
aster WHERE type='table'").fetchall())
> print('PRAGMA show_tables db:', con.execute("SELECT * FROM db.main.sqlite_
master WHERE type='table'").fetchall())
> PY
PRAGMA show_tables main: [('table', 't', 't', 0, 'CREATE TABLE t(a BIGINT);')]
Traceback (most recent call last):
File "<stdin>", line 13, in <module>
duckdb.duckdb.CatalogException: Catalog Error: Table with name sqlite_master doe
s not exist!
Did you mean "main.sqlite_master or memory.sqlite_master"?
LINE 1: SELECT * FROM db.main.sqlite_master WHERE type='table'
^
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck#The database is mounted as 'db'. I will attach it properly and query the tables.
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck# python - <<'PY'
> import duckdb, sqlite3, tempfile, pathlib
>
> with tempfile.TemporaryDirectory() as td:
> path = pathlib.Path(td)/'test.sqlite'
> conn = sqlite3.connect(path)
> conn.execute('CREATE TABLE t(a int)')
> conn.commit(); conn.close()
>
> con = duckdb.connect()
> con.execute('LOAD sqlite')
> con.execute(f"ATTACH '{path}' (TYPE SQLITE)")
> print('tables from main.sqlite_master', con.execute("SELECT name FROM main
.sqlite_master WHERE type='table'").fetchall())
> print('tables from sqlite_master', con.execute("SELECT name FROM sqlite_ma
ster WHERE type='table'").fetchall())
> con.execute('CREATE VIEW t_view AS SELECT * FROM main.t')
> print('Show tables', con.execute('SHOW TABLES').fetchall())
> PY
tables from main.sqlite_master [('t',)]
tables from sqlite_master [('t',)]
Traceback (most recent call last):
File "<stdin>", line 14, in <module>
duckdb.duckdb.CatalogException: Catalog Error: Table with name t does not exist!
Did you mean "test.t"?
LINE 1: CREATE VIEW t_view AS SELECT * FROM main.t
^
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck#I'm wondering if the alias 'test' is auto-created from the file name. Let's verify.
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck# python - <<'PY'
> import duckdb, sqlite3, tempfile, pathlib
>
> with tempfile.TemporaryDirectory() as td:
> path = pathlib.Path(td)/'mydb.sqlite'
> conn = sqlite3.connect(path)
> conn.execute('CREATE TABLE t(a int)')
> conn.commit(); conn.close()
>
> con = duckdb.connect()
> con.execute('LOAD sqlite')
> con.execute(f"ATTACH '{path}' (TYPE SQLITE)")
> print('SHOW SCHEMAS -> maybe not working but try PRAGMA?', con.execute("PR
AGMA database_list").fetchall())
> print('SHOW TABLES maybe from main schema? ->', con.execute('PRAGMA show_t
ables;').fetchall())
> PY
SHOW SCHEMAS -> maybe not working but try PRAGMA? [(546, 'memory', None), (1944,
'mydb', '/tmp/tmpws9dbphp/mydb.sqlite')]
SHOW TABLES maybe from main schema? -> []
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck#Looks like tables are accessed as mydb.mytable, but 'sqlite_master' uses the default schema.
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck# python - <<'PY'
> import duckdb, sqlite3, tempfile, pathlib
>
> with tempfile.TemporaryDirectory() as td:
> path = pathlib.Path(td)/'mydb.sqlite'
> conn = sqlite3.connect(path)
> conn.execute('CREATE TABLE t(a int)')
> conn.commit(); conn.close()
>
> con = duckdb.connect()
> con.execute('LOAD sqlite')
> con.execute(f"ATTACH '{path}' (TYPE SQLITE)")
> print('PRAGMA database_list ->', con.execute('PRAGMA database_list').fetch
all())
> print('select from mydb.main.sqlite_master ->', con.execute('SELECT name F
ROM mydb.main.sqlite_master WHERE type="table"').fetchall())
> print('select from mydb.sqlite_master ->', con.execute('SELECT name FROM m
ydb.sqlite_master WHERE type="table"').fetchall())
> print('select from main.sqlite_master ->', con.execute('SELECT name FROM m
ain.sqlite_master WHERE type="table"').fetchall())
> print('select from sqlite_master ->', con.execute('SELECT name FROM sqlite
_master WHERE type="table"').fetchall())
> print('select from mydb.main.t -> maybe table to query', con.execute('SELE
CT * FROM mydb.t').fetchall())
> PY
PRAGMA database_list -> [(546, 'memory', None), (1944, 'mydb', '/tmp/tmp5air25e5
/mydb.sqlite')]
Traceback (most recent call last):
File "<stdin>", line 13, in <module>
duckdb.duckdb.CatalogException: Catalog Error: Table with name sqlite_master doe
s not exist!
Did you mean "main.sqlite_master or memory.sqlite_master"?
LINE 1: SELECT name FROM mydb.main.sqlite_master WHERE type="table"
^
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck#I realized the schema alias is 'mydb', not 'mydb.main'. I'll select from mydb.sqlite_master.
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck# python - <<'PY'
> import duckdb, sqlite3, tempfile, pathlib
>
> with tempfile.TemporaryDirectory() as td:
> path = pathlib.Path(td)/'mydb.sqlite'
> conn = sqlite3.connect(path)
> conn.execute('CREATE TABLE t(a int)')
> conn.commit(); conn.close()
>
> con = duckdb.connect()
> con.execute('LOAD sqlite')
> con.execute(f"ATTACH '{path}' (TYPE SQLITE)")
> print('select from mydb.sqlite_master ->', con.execute('SELECT name FROM m
ydb.sqlite_master WHERE type="table"').fetchall())
> print('select from main.sqlite_master ->', con.execute('SELECT name FROM m
ain.sqlite_master WHERE type="table"').fetchall())
> print('select from sqlite_master ->', con.execute('SELECT name FROM sqlite
_master WHERE type="table"').fetchall())
> PY
Traceback (most recent call last):
File "<stdin>", line 12, in <module>
duckdb.duckdb.CatalogException: Catalog Error: Table with name sqlite_master doe
s not exist!
Did you mean "main.sqlite_master or memory.sqlite_master"?
LINE 1: SELECT name FROM mydb.sqlite_master WHERE type="table"
^
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck#I couldn't find 'sqlite_master'. I'll double-check with "SELECT name FROM mydb.sqlite_master".
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck# python - <<'PY'
> import duckdb, sqlite3, tempfile, pathlib
>
> with tempfile.TemporaryDirectory() as td:
> path = pathlib.Path(td)/'mydb.sqlite'
> conn = sqlite3.connect(path)
> conn.execute('CREATE TABLE t(a int)')
> conn.commit(); conn.close()
>
> con = duckdb.connect()
> con.execute('LOAD sqlite')
> con.execute(f"ATTACH '{path}' AS db (TYPE SQLITE)")
> print('PRAGMA database_list ->', con.execute('PRAGMA database_list').fetch
all())
> print('SELECT * FROM db.sqlite_master ->', con.execute('SELECT name FROM d
b.sqlite_master WHERE type="table"').fetchall())
> print('SELECT * FROM main.sqlite_master ->', con.execute('SELECT name FROM
main.sqlite_master WHERE type="table"').fetchall())
> print('SELECT name FROM sqlite_master ->', con.execute('SELECT name FROM s
qlite_master WHERE type="table"').fetchall())
> print('select from db.t ->', con.execute('SELECT * FROM db.t').fetchall())
> PY
PRAGMA database_list -> [(546, 'memory', None), (1944, 'db', '/tmp/tmpxgtolwsl/m
ydb.sqlite')]
Traceback (most recent call last):
File "<stdin>", line 13, in <module>
duckdb.duckdb.CatalogException: Catalog Error: Table with name sqlite_master doe
s not exist!
Did you mean "main.sqlite_master or memory.sqlite_master"?
LINE 1: SELECT name FROM db.sqlite_master WHERE type="table"
^
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck#I'll try running SELECT * FROM db.t to test the query.
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck# python - <<'PY'
> import duckdb, sqlite3, tempfile, pathlib
>
> with tempfile.TemporaryDirectory() as td:
> path = pathlib.Path(td)/'db.sqlite'
> conn = sqlite3.connect(path)
> conn.execute('CREATE TABLE t(a int)'); conn.commit(); conn.close()
>
> con = duckdb.connect()
> con.execute('LOAD sqlite')
> con.execute(f"ATTACH '{path}' AS db (TYPE SQLITE)")
> print('SELECT name FROM main.sqlite_master ->', con.execute('SELECT name F
ROM main.sqlite_master WHERE type="table"').fetchall())
> print('SELECT * FROM db.t ->', con.execute('SELECT * FROM db.t').fetchall(
))
> con.execute('CREATE VIEW t_copy AS SELECT * FROM db.t')
> print('SHOW TABLES ->', con.execute('SHOW TABLES').fetchall())
> PY
Traceback (most recent call last):
File "<stdin>", line 11, in <module>
duckdb.duckdb.BinderException: Binder Error: Referenced column "table" not found
in FROM clause!
Candidate bindings: "tbl_name", "type", "name", "rootpage", "sql"
LINE 1: SELECT name FROM main.sqlite_master WHERE type="table"
^
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck#I wrote the wrong query for checking tables. I'll fix the column names and retry.
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck# python - <<'PY'
> import duckdb, sqlite3, tempfile, pathlib
>
> with tempfile.TemporaryDirectory() as td:
> path = pathlib.Path(td)/'db.sqlite'
> conn = sqlite3.connect(path)
> conn.execute('CREATE TABLE t(a int)')
> conn.commit(); conn.close()
>
> con = duckdb.connect()
> con.execute('LOAD sqlite')
> con.execute(f"ATTACH '{path}' AS db (TYPE SQLITE)")
> print('select from main.sqlite_master', con.execute('SELECT name FROM main
.sqlite_master WHERE type = ''table''').fetchall())
> PY
Traceback (most recent call last):
File "<stdin>", line 12, in <module>
duckdb.duckdb.ParserException: Parser Error: syntax error at or near "table"
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck#I'll use triple quotes instead of single quotes for proper string formatting.
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck# python - <<'PY'
> import duckdb, sqlite3, tempfile, pathlib
>
> with tempfile.TemporaryDirectory() as td:
> path = pathlib.Path(td)/'db.sqlite'
> conn = sqlite3.connect(path)
> conn.execute('CREATE TABLE t(a int)')
> conn.commit(); conn.close()
>
> con = duckdb.connect()
> con.execute('LOAD sqlite')
> con.execute(f"ATTACH '{path}' AS db (TYPE SQLITE)")
> print('select from main.sqlite_master', con.execute("SELECT name FROM main
.sqlite_master WHERE type='table'").fetchall())
> print('select from sqlite_master', con.execute("SELECT name FROM sqlite_ma
ster WHERE type='table'").fetchall())
> print('select from db.t', con.execute('SELECT * FROM db.t').fetchall())
> con.execute('CREATE VIEW t_copy AS SELECT * FROM db.t')
> print('SHOW TABLES', con.execute('SHOW TABLES').fetchall())
> PY
select from main.sqlite_master [('t',)]
select from sqlite_master [('t',)]
select from db.t []
SHOW TABLES [('t_copy',)]
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck#The table t doesn't exist in the DuckDB database. I realized I didn't insert any rows. I'll fix the queries to check main.sqlite_master for the table list.
erver.py
return "VARCHAR"
if "BLOB" in t:
return "BLOB"
if any(key in t for key in ("DOUBLE", "REAL", "FLOA", "NUMERIC", "DECIMAL"))
:
return "DOUBLE"
if "BOOL" in t:
return "BOOLEAN"
if "DATE" in t or "TIME" in t:
return "TIMESTAMP" if "TIME" in t else "DATE"
return "VARCHAR"
def _load_database(path: Path) -> duckdb.DuckDBPyConnection:
if not path.exists():
raise FileNotFoundError(path)
ext = path.suffix.lower()
if ext == ".csv":
con = duckdb.connect()
con.execute(
f"CREATE TABLE events AS SELECT * FROM read_csv_auto('{path.as_posix
()}')"
)
elif ext in {".db", ".sqlite"}:
con = duckdb.connect()
try:
con.execute("LOAD sqlite")
con.execute(f"ATTACH '{path.as_posix()}' AS db (TYPE SQLITE)")
tables = [
r[0]
for r in con.execute(
"SELECT name FROM db.sqlite_master WHERE type='table'"
).fetchall()
]
for t in tables:
con.execute(f'CREATE VIEW "{t}" AS SELECT * FROM db."{t}"')
except Exception:
sconn = sqlite3.connect(path)
tables = [
r[0]
for r in sconn.execute(
"SELECT name FROM sqlite_master WHERE type='table'"
).fetchall()
]
for t in tables:
info = sconn.execute(f'PRAGMA table_info("{t}")').fetchall()
col_defs = ", ".join(
f'"{r[1]}" {_normalize_sqlite_type(cast(str, r[2]))}' for r
in info
)
sql = f'CREATE TABLE "{t}" ({col_defs})'
try:
con.execute(sql)
except Exception:
print(f"Failed SQL: {sql}")
raise
placeholders = ",".join("?" for _ in info)
for row in sconn.execute(f'SELECT * FROM "{t}"'):
con.execute(f'INSERT INTO "{t}" VALUES ({placeholders})', ro
w)
sconn.close()
else:
con = duckdb.connect(path)
return con
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck#I’ll update the query to directly use sqlite_master. No alias needed after attaching. Let's check possible conflicts with DuckDB’s memory.
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck# python - <<'PY'
> import duckdb
>
> con = duckdb.connect()
> try:
> print('sqlite_master check', con.execute("SELECT * FROM sqlite_master").fe
tchall())
> except Exception as e:
> print('error', e)
> PY
sqlite_master check []
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck#The sqlite_master table isn’t empty, but it’s unclear why it's showing no results.
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck# python - <<'PY'
> import duckdb
>
> con = duckdb.connect()
> print(con.execute('SHOW TABLES').fetchall())
> PY
[]
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck#I initially thought to use sqlite_master, but found that the real schema is "main".
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck# python - <<'PY'
> import duckdb, sqlite3, tempfile, pathlib
>
> with tempfile.TemporaryDirectory() as td:
> path = pathlib.Path(td)/'mydb.sqlite'
> conn = sqlite3.connect(path)
> conn.execute('CREATE TABLE t(a int)')
> conn.execute('CREATE TABLE u(b int)')
> conn.commit(); conn.close()
>
> con = duckdb.connect()
> con.execute('LOAD sqlite')
> con.execute(f"ATTACH '{path}' AS db (TYPE SQLITE)")
> print('SHOW TABLES FROM db ->', con.execute('SHOW TABLES FROM db').fetchal
l())
> print('SHOW TABLES FROM main ->', con.execute('SHOW TABLES FROM main').fet
chall())
> PY
Traceback (most recent call last):
File "<stdin>", line 13, in <module>
duckdb.duckdb.ParserException: Parser Error: syntax error at or near "FROM"
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck#I can query sqlite_master for table names instead of using SHOW TABLES.
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck# python - <<'PY'
> import duckdb, sqlite3, tempfile, pathlib
>
> with tempfile.TemporaryDirectory() as td:
> path = pathlib.Path(td)/'db.sqlite'
> conn = sqlite3.connect(path)
> conn.execute('CREATE TABLE t(a int)'); conn.execute('CREATE TABLE u(b int)
')
> conn.commit(); conn.close()
>
> con = duckdb.connect()
> con.execute('LOAD sqlite')
> con.execute(f"ATTACH '{path}' AS db (TYPE SQLITE)")
> print('sqlite_master ->', con.execute("SELECT name FROM sqlite_master WHER
E type='table'").fetchall())
> con.execute('CREATE VIEW t_copy AS SELECT * FROM db.t')
> print('SHOW TABLES ->', con.execute('SHOW TABLES').fetchall())
> PY
sqlite_master -> [('t',), ('u',)]
SHOW TABLES -> [('t_copy',)]
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck#I need to modify tests related to the fallback path, like test_sqlite_longvarchar.
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck# grep -n "failing_connect" -n
tests/test_server.py
227: def failing_connect(*args: Any, **kwargs: Any) -> Any:
246: monkeypatch.setattr(server.duckdb, "connect", failing_connect)
283: def failing_connect(*args: Any, **kwargs: Any) -> Any:
302: monkeypatch.setattr(server.duckdb, "connect", failing_connect)(scubaduck) root@3a541e64f0a7:/workspace/scubaduck# sed -n '210,260p' tests/test
_server.py
sqlite_file = tmp_path / "events.sqlite"
import sqlite3
conn = sqlite3.connect(sqlite_file)
conn.execute(
"CREATE TABLE events (timestamp TEXT, url LONGVARCHAR, title VARCHAR(10)
)"
)
conn.execute(
"INSERT INTO events VALUES ('2024-01-01 00:00:00', 'https://a.com', 'Hom
e')"
)
conn.commit()
conn.close() # pyright: ignore[reportUnknownMemberType, reportAttributeAcce
ssIssue]
from typing import Any
real_connect = duckdb.connect
def failing_connect(*args: Any, **kwargs: Any) -> Any:
real = real_connect(*args, **kwargs)
class Wrapper:
def __init__(self, con: duckdb.DuckDBPyConnection) -> None:
self.con = con
self._failed = False
def execute(self, sql: str, *a: Any, **kw: Any):
if not self._failed and sql == "LOAD sqlite":
self._failed = True
raise RuntimeError("fail")
return self.con.execute(sql, *a, **kw)
def __getattr__(self, name: str) -> object:
return getattr(self.con, name)
return Wrapper(real)
monkeypatch.setattr(server.duckdb, "connect", failing_connect)
app = server.create_app(sqlite_file)
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-01 01:00:00",
"order_by": "timestamp",
"columns": ["timestamp", "url", "title"],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()_server.py
data = rv.get_json()
assert rv.status_code == 200
assert data["rows"][0][1] == "https://a.com"
def test_sqlite_bigint(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
sqlite_file = tmp_path / "big.sqlite"
import sqlite3
conn = sqlite3.connect(sqlite_file)
conn.execute("CREATE TABLE events (timestamp TEXT, value INTEGER)")
big_value = 13385262862605259
conn.execute(
"INSERT INTO events VALUES ('2024-01-01 00:00:00', ?)",
(big_value,),
)
conn.commit()
conn.close() # pyright: ignore[reportUnknownMemberType, reportAttributeAcce
ssIssue]
from typing import Any
real_connect = duckdb.connect
def failing_connect(*args: Any, **kwargs: Any) -> Any:
real = real_connect(*args, **kwargs)
class Wrapper:
def __init__(self, con: duckdb.DuckDBPyConnection) -> None:
self.con = con
self._failed = False
def execute(self, sql: str, *a: Any, **kw: Any):
if not self._failed and sql == "LOAD sqlite":
self._failed = True
raise RuntimeError("fail")
return self.con.execute(sql, *a, **kw)
def __getattr__(self, name: str) -> object:
return getattr(self.con, name)
return Wrapper(real)
monkeypatch.setattr(server.duckdb, "connect", failing_connect)
app = server.create_app(sqlite_file)
client = app.test_client()
payload = {
"table": "events",
"order_by": "timestamp",
"columns": ["timestamp", "value"],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert data["rows"][0][1] == big_value
def test_integer_time_column(tmp_path: Path) -> None:
csv_file = tmp_path / "events.csv"
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck#I'll remove the fallback path and update tests for LONGVARCHAR and BIGINT without monkeypatching.
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck# grep -n "monkeypatch.setattr
(server.duckdb" -n tests/test_server.py
246: monkeypatch.setattr(server.duckdb, "connect", failing_connect)
302: monkeypatch.setattr(server.duckdb, "connect", failing_connect)
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck#I'll update the tests and modify _load_database as shown, and remove the sqlite3 import.
py
from __future__ import annotations
from dataclasses import dataclass, field, replace
from typing import Any, Dict, List, Tuple, cast
import re
from datetime import datetime, timedelta, timezone
import time
from pathlib import Path
import os
import sqlite3
import traceback
import math
import duckdb
from dateutil import parser as dtparser
from flask import Flask, jsonify, request, send_from_directory
@dataclass
class Filter:
column: str
op: str
value: str | int | float | list[str] | None
@dataclass
class QueryParams:
start: str | None = None
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck#I'll search for any sqlite3 usage in server.py before removing it.
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck# grep -n "sqlite3" -n scubadu
ck/server.py
12:import sqlite3
96: sconn = sqlite3.connect(path)
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck#I'll remove the sqlite3 import and inspect the pyproject for ruff and pyright settings.
name = "scubaduck"
version = "0.1.0"
description = "Scuba on DuckDB"
readme = "README.md"
requires-python = ">=3.12"
license = {text = "MIT"}
dependencies = [
"duckdb",
"flask",
"python-dateutil>=2.9.0.post0",
]
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"
[tool.pytest.ini_options]
addopts = "-n auto --tb=native"
[tool.pyright]
# Pyright configuration with strict settings
include = ["scubaduck", "tests"]
exclude = ["**/__pycache__", "dist"]
stubPath = "stubs"
venvPath = "."
venv = ".venv"
reportMissingImports = true
reportMissingTypeStubs = true
pythonVersion = "3.12"
pythonPlatform = "All"
typeCheckingMode = "strict"
reportUnknownMemberType = true
reportUnknownParameterType = true
reportUnknownVariableType = true
reportUnknownArgumentType = true
reportPrivateImportUsage = true
reportUntypedFunctionDecorator = true
reportFunctionMemberAccess = true
reportIncompatibleMethodOverride = true
[tool.ruff]
target-version = "py312"
[tool.uv]
dev-dependencies = [
"pytest>=8.3.5",
"ruff>=0.11.2",
"pytest-xdist>=3.6.1",
"playwright>=1.52.0",
"pyright>=1.1.350",
"pytest-playwright>=0.7.0",
]
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck#I'll update server.py to be compatible with Python 3.12 as required by Ruff.
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck# apply_patch<<'PATCH'
> *** Begin Patch
> *** Update File: scubaduck/server.py
> @@
> -import sqlite3
> *** End Patch
> PATCH
Success. Updated the following files:
M scubaduck/server.py
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck#I'm going to make adjustments to the _load_database function now.
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck# sed -n '74,118p' scubaduck/s
erver.py
ext = path.suffix.lower()
if ext == ".csv":
con = duckdb.connect()
con.execute(
f"CREATE TABLE events AS SELECT * FROM read_csv_auto('{path.as_posix
()}')"
)
elif ext in {".db", ".sqlite"}:
con = duckdb.connect()
try:
con.execute("LOAD sqlite")
con.execute(f"ATTACH '{path.as_posix()}' AS db (TYPE SQLITE)")
tables = [
r[0]
for r in con.execute(
"SELECT name FROM db.sqlite_master WHERE type='table'"
).fetchall()
]
for t in tables:
con.execute(f'CREATE VIEW "{t}" AS SELECT * FROM db."{t}"')
except Exception:
sconn = sqlite3.connect(path)
tables = [
r[0]
for r in sconn.execute(
"SELECT name FROM sqlite_master WHERE type='table'"
).fetchall()
]
for t in tables:
info = sconn.execute(f'PRAGMA table_info("{t}")').fetchall()
col_defs = ", ".join(
f'"{r[1]}" {_normalize_sqlite_type(cast(str, r[2]))}' for r
in info
)
sql = f'CREATE TABLE "{t}" ({col_defs})'
try:
con.execute(sql)
except Exception:
print(f"Failed SQL: {sql}")
raise
placeholders = ",".join("?" for _ in info)
for row in sconn.execute(f'SELECT * FROM "{t}"'):
con.execute(f'INSERT INTO "{t}" VALUES ({placeholders})', ro
w)
sconn.close()
else:
con = duckdb.connect(path)
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck#I’ll go ahead and update the code to remove the fallback mechanism.
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck# apply_patch<<'PATCH'
> *** Begin Patch
> *** Update File: scubaduck/server.py
> @@
> - elif ext in {".db", ".sqlite"}:
> - con = duckdb.connect()
> - try:
> - con.execute("LOAD sqlite")
> - con.execute(f"ATTACH '{path.as_posix()}' AS db (TYPE SQLITE)")
> - tables = [
> - r[0]
> - for r in con.execute(
> - "SELECT name FROM db.sqlite_master WHERE type='table'"
> - ).fetchall()
> - ]
> - for t in tables:
> - con.execute(f'CREATE VIEW "{t}" AS SELECT * FROM db."{t}"')
> - except Exception:
> - sconn = sqlite3.connect(path)
> - tables = [
> - r[0]
> - for r in sconn.execute(
> - "SELECT name FROM sqlite_master WHERE type='table'"
> - ).fetchall()
> - ]
> - for t in tables:
> - info = sconn.execute(f'PRAGMA table_info("{t}")').fetchall()
> - col_defs = ", ".join(
> - f'"{r[1]}" {_normalize_sqlite_type(cast(str, r[2]))}' for
r in info
> - )
> - sql = f'CREATE TABLE "{t}" ({col_defs})'
> - try:
> - con.execute(sql)
> - except Exception:
> - print(f"Failed SQL: {sql}")
> - raise
> - placeholders = ",".join("?" for _ in info)
> - for row in sconn.execute(f'SELECT * FROM "{t}"'):
> - con.execute(f'INSERT INTO "{t}" VALUES ({placeholders})',
row)
> - sconn.close()
> + elif ext in {".db", ".sqlite"}:
> + con = duckdb.connect()
> + con.execute("LOAD sqlite")
> + con.execute(f"ATTACH '{path.as_posix()}' AS db (TYPE SQLITE)")
> + tables = [
> + r[0]
> + for r in con.execute(
> + "SELECT name FROM sqlite_master WHERE type='table'"
> + ).fetchall()
> + ]
> + for t in tables:
> + con.execute(f'CREATE VIEW "{t}" AS SELECT * FROM db."{t}"')
> *** End Patch
> PATCH
Success. Updated the following files:
M scubaduck/server.py
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck#I think I missed a closing bracket after the patch. Let me check for leftover lines.
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck# sed -n '74,112p' scubaduck/s
erver.py
ext = path.suffix.lower()
if ext == ".csv":
con = duckdb.connect()
con.execute(
f"CREATE TABLE events AS SELECT * FROM read_csv_auto('{path.as_posix
()}')"
)
elif ext in {".db", ".sqlite"}:
con = duckdb.connect()
con.execute("LOAD sqlite")
con.execute(f"ATTACH '{path.as_posix()}' AS db (TYPE SQLITE)")
tables = [
r[0]
for r in con.execute(
"SELECT name FROM sqlite_master WHERE type='table'"
).fetchall()
]
for t in tables:
con.execute(f'CREATE VIEW "{t}" AS SELECT * FROM db."{t}"')
else:
con = duckdb.connect(path)
return con
_REL_RE = re.compile(
r"([+-]?\d+(?:\.\d*)?)\s*(hour|hours|day|days|week|weeks|fortnight|fortnight
s)",
re.IGNORECASE,
)
def parse_time(val: str | None) -> str | None:
"""Parse an absolute or relative time string into ``YYYY-MM-DD HH:MM:SS``.""
"
if val is None or val == "":
return None
s = val.strip()
if s.lower() == "now":
dt = datetime.now(timezone.utc)
return dt.replace(microsecond=0).strftime("%Y-%m-%d %H:%M:%S")
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck#I’ll clean up the leftover sqlite3 import and run ruff format.
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck# ruff format scubaduck/server
.py
1 file left unchangedpy
All checks passed!
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck#I’ll edit tests to remove the monkeypatch and check lines 210-330 for necessary changes.
sqlite_file = tmp_path / "events.sqlite"
import sqlite3
conn = sqlite3.connect(sqlite_file)
conn.execute(
"CREATE TABLE events (timestamp TEXT, url LONGVARCHAR, title VARCHAR(10)
)"
)
conn.execute(
"INSERT INTO events VALUES ('2024-01-01 00:00:00', 'https://a.com', 'Hom
e')"
)
conn.commit()
conn.close() # pyright: ignore[reportUnknownMemberType, reportAttributeAcce
ssIssue]
from typing import Any
real_connect = duckdb.connect
def failing_connect(*args: Any, **kwargs: Any) -> Any:
real = real_connect(*args, **kwargs)
class Wrapper:
def __init__(self, con: duckdb.DuckDBPyConnection) -> None:
self.con = con
self._failed = False
def execute(self, sql: str, *a: Any, **kw: Any):
if not self._failed and sql == "LOAD sqlite":
self._failed = True
raise RuntimeError("fail")
return self.con.execute(sql, *a, **kw)
def __getattr__(self, name: str) -> object:
return getattr(self.con, name)
return Wrapper(real)
monkeypatch.setattr(server.duckdb, "connect", failing_connect)
app = server.create_app(sqlite_file)
client = app.test_client()
payload = {
"table": "events",
"start": "2024-01-01 00:00:00",
"end": "2024-01-01 01:00:00",
"order_by": "timestamp",
"columns": ["timestamp", "url", "title"],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert data["rows"][0][1] == "https://a.com"
def test_sqlite_bigint(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
sqlite_file = tmp_path / "big.sqlite"
import sqlite3
conn = sqlite3.connect(sqlite_file)
conn.execute("CREATE TABLE events (timestamp TEXT, value INTEGER)")
big_value = 13385262862605259
conn.execute(
"INSERT INTO events VALUES ('2024-01-01 00:00:00', ?)",
(big_value,),
)
conn.commit()
conn.close() # pyright: ignore[reportUnknownMemberType, reportAttributeAcce
ssIssue]
from typing import Any
real_connect = duckdb.connect
def failing_connect(*args: Any, **kwargs: Any) -> Any:
real = real_connect(*args, **kwargs)
class Wrapper:
def __init__(self, con: duckdb.DuckDBPyConnection) -> None:
self.con = con
self._failed = False
def execute(self, sql: str, *a: Any, **kw: Any):
if not self._failed and sql == "LOAD sqlite":
self._failed = True
raise RuntimeError("fail")
return self.con.execute(sql, *a, **kw)
def __getattr__(self, name: str) -> object:
return getattr(self.con, name)
return Wrapper(real)
monkeypatch.setattr(server.duckdb, "connect", failing_connect)
app = server.create_app(sqlite_file)
client = app.test_client()
payload = {
"table": "events",
"order_by": "timestamp",
"columns": ["timestamp", "value"],
}
rv = client.post(
"/api/query", data=json.dumps(payload), content_type="application/json"
)
data = rv.get_json()
assert rv.status_code == 200
assert data["rows"][0][1] == big_value
def test_integer_time_column(tmp_path: Path) -> None:
csv_file = tmp_path / "events.csv"
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck#I'll adjust the tests to avoid patching and just execute simple queries, checking results.
> *** Begin Patch
> *** Update File: tests/test_server.py
> @@
> -def test_sqlite_longvarchar(tmp_path: Path, monkeypatch: pytest.MonkeyPatch)
-> None:
> +def test_sqlite_longvarchar(tmp_path: Path) -> None:
> @@
> - from typing import Any
> -
> - real_connect = duckdb.connect
> -
> - def failing_connect(*args: Any, **kwargs: Any) -> Any:
> - real = real_connect(*args, **kwargs)
> -
> - class Wrapper:
> - def __init__(self, con: duckdb.DuckDBPyConnection) -> None:
> - self.con = con
> - self._failed = False
> -
> - def execute(self, sql: str, *a: Any, **kw: Any):
> - if not self._failed and sql == "LOAD sqlite":
> - self._failed = True
> - raise RuntimeError("fail")
> - return self.con.execute(sql, *a, **kw)
> -
> - def __getattr__(self, name: str) -> object:
> - return getattr(self.con, name)
> -
> - return Wrapper(real)
> -
> - monkeypatch.setattr(server.duckdb, "connect", failing_connect)
> -
> app = server.create_app(sqlite_file)
> *** End Patch
> PATCH
Success. Updated the following files:
M tests/test_server.py
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck#I'll update the test_sqlite_bigint for the required changes.
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck# apply_patch<<'PATCH'
> *** Begin Patch
> *** Update File: tests/test_server.py
> @@
> -def test_sqlite_bigint(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> No
ne:
> +def test_sqlite_bigint(tmp_path: Path) -> None:
> @@
> - from typing import Any
> -
> - real_connect = duckdb.connect
> -
> - def failing_connect(*args: Any, **kwargs: Any) -> Any:
> - real = real_connect(*args, **kwargs)
> -
> - class Wrapper:
> - def __init__(self, con: duckdb.DuckDBPyConnection) -> None:
> - self.con = con
> - self._failed = False
> -
> - def execute(self, sql: str, *a: Any, **kw: Any):
> - if not self._failed and sql == "LOAD sqlite":
> - self._failed = True
> - raise RuntimeError("fail")
> - return self.con.execute(sql, *a, **kw)
> -
> - def __getattr__(self, name: str) -> object:
> - return getattr(self.con, name)
> -
> - return Wrapper(real)
> -
> - monkeypatch.setattr(server.duckdb, "connect", failing_connect)
> -
> app = server.create_app(sqlite_file)
> *** End Patch
> PATCH
Success. Updated the following files:
M tests/test_server.py
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck#I'll run both the ruff formatter and the check command for the test file.
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck# ruff format tests/test_serve
r.py
1 file left unchanged(scubaduck) root@3a541e64f0a7:/workspace/scubaduck# ruff check tests/test_server
.py
All checks passed!
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck#I'll run pyright to check for any typing issues.
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck# pyright/workspace/scubaduck/scubaduck/server.py
/workspace/scubaduck/scubaduck/server.py:49:5 - error: Function "_normalize_sq
lite_type" is not accessed (reportUnusedFunction)
1 error, 0 warnings, 0 informations
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck#I'll search for _normalize_sqlite_type to see if it can be removed.
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck# grep -n "_normalize_sqlite_t
ype" -n scubaduck -r
scubaduck/server.py:49:def _normalize_sqlite_type(sql: str) -> str:
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck#Since the function isn't used, I'll patch it by removing both its definition and any calls.
rver.py
show_hits: bool = False
x_axis: str | None = None
granularity: str = "Auto"
fill: str = "0"
table: str = "events"
time_column: str = "timestamp"
time_unit: str = "s"
def _normalize_sqlite_type(sql: str) -> str:
"""Map arbitrary SQLite column types to DuckDB-compatible types."""
t = sql.strip().upper()
if "(" in t:
t = t.split("(", 1)[0]
if "INT" in t:
# SQLite only has a single INTEGER type which is always 64-bit.
# Use DuckDB's BIGINT to avoid overflow when values exceed INT32.
return "BIGINT"
if any(key in t for key in ("CHAR", "CLOB", "TEXT")):
return "VARCHAR"
if "BLOB" in t:
return "BLOB"
if any(key in t for key in ("DOUBLE", "REAL", "FLOA", "NUMERIC", "DECIMAL"))
:
return "DOUBLE"
if "BOOL" in t:
return "BOOLEAN"
if "DATE" in t or "TIME" in t:
return "TIMESTAMP" if "TIME" in t else "DATE"
return "VARCHAR"
def _load_database(path: Path) -> duckdb.DuckDBPyConnection:
if not path.exists():
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck#I'm deleting the _normalize_sqlite_type function entirely.
> *** Update File: scubaduck/server.py
> @@
> - time_unit: str = "s"
> -
> -
> -def _normalize_sqlite_type(sql: str) -> str:
> - """Map arbitrary SQLite column types to DuckDB-compatible types."""
> - t = sql.strip().upper()
> - if "(" in t:
> - t = t.split("(", 1)[0]
> - if "INT" in t:
> - # SQLite only has a single INTEGER type which is always 64-bit.
> - # Use DuckDB's BIGINT to avoid overflow when values exceed INT32.
> - return "BIGINT"
> - if any(key in t for key in ("CHAR", "CLOB", "TEXT")):
> - return "VARCHAR"
> - if "BLOB" in t:
> - return "BLOB"
> - if any(key in t for key in ("DOUBLE", "REAL", "FLOA", "NUMERIC", "DECIMAL
")):
> - return "DOUBLE"
> - if "BOOL" in t:
> - return "BOOLEAN"
> - if "DATE" in t or "TIME" in t:
> - return "TIMESTAMP" if "TIME" in t else "DATE"
> - return "VARCHAR"
> + time_unit: str = "s"
> *** End Patch
> PATCH
Success. Updated the following files:
M scubaduck/server.py
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck#I’ll add a blank line after the dataclass to keep things clean.
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck# sed -n '48,82p' scubaduck/se
rver.py
def _load_database(path: Path) -> duckdb.DuckDBPyConnection:
if not path.exists():
raise FileNotFoundError(path)
ext = path.suffix.lower()
if ext == ".csv":
con = duckdb.connect()
con.execute(
f"CREATE TABLE events AS SELECT * FROM read_csv_auto('{path.as_posix
()}')"
)
elif ext in {".db", ".sqlite"}:
con = duckdb.connect()
con.execute("LOAD sqlite")
con.execute(f"ATTACH '{path.as_posix()}' AS db (TYPE SQLITE)")
tables = [
r[0]
for r in con.execute(
"SELECT name FROM sqlite_master WHERE type='table'"
).fetchall()
]
for t in tables:
con.execute(f'CREATE VIEW "{t}" AS SELECT * FROM db."{t}"')
else:
con = duckdb.connect(path)
return con
_REL_RE = re.compile(
r"([+-]?\d+(?:\.\d*)?)\s*(hour|hours|day|days|week|weeks|fortnight|fortnight
s)",
re.IGNORECASE,
)
def parse_time(val: str | None) -> str | None:
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck#I need to review previous lines to see if I accidentally removed any dataclass code.
class Filter:
column: str
op: str
value: str | int | float | list[str] | None
@dataclass
class QueryParams:
start: str | None = None
end: str | None = None
order_by: str | None = None
order_dir: str = "ASC"
limit: int | None = None
columns: list[str] = field(default_factory=lambda: [])
filters: list[Filter] = field(default_factory=lambda: [])
derived_columns: dict[str, str] = field(default_factory=lambda: {})
graph_type: str = "samples"
group_by: list[str] = field(default_factory=lambda: [])
aggregate: str | None = None
show_hits: bool = False
x_axis: str | None = None
granularity: str = "Auto"
fill: str = "0"
table: str = "events"
time_column: str = "timestamp"
time_unit: str = "s"
def _load_database(path: Path) -> duckdb.DuckDBPyConnection:
if not path.exists():
raise FileNotFoundError(path)
ext = path.suffix.lower()
if ext == ".csv":
con = duckdb.connect()
con.execute(
f"CREATE TABLE events AS SELECT * FROM read_csv_auto('{path.as_posix
()}')"
)
elif ext in {".db", ".sqlite"}:
con = duckdb.connect()
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck#I removed the _normalize_sqlite_type lines and left a blank line; let's run ruff format.
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck# ruff format scubaduck/server
.py tests/test_server.py
2 files left unchanged(scubaduck) root@3a541e64f0a7:/workspace/scubaduck# ruff check scubaduck/server.
py tests/test_server.py
All checks passed!
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck#I'll try running pyright once again to check for any issues.
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck# pyright0 errors, 0 warnings, 0 informations
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck#I'll go ahead and execute the tests using pytest now.
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck# pytest -q........................................................................ [ 78%].................... [100%]
92 passed in 11.99s
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck#All tests passed. I'll check the git status to confirm everything is up to date.
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck# git status --short
M scubaduck/server.py
M tests/test_server.py
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck#Time to stage and commit, using the conventional commit message format.
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck# git add scubaduck/server.py
tests/test_server.py(scubaduck) root@3a541e64f0a7:/workspace/scubaduck# git commit -m "fix(server):
remove sqlite fallback" && git status --short
[work 6445ee7] fix(server): remove sqlite fallback
2 files changed, 12 insertions(+), 109 deletions(-)(scubaduck) root@3a541e64f0a7:/workspace/scubaduck# git status --short(scubaduck) root@3a541e64f0a7:/workspace/scubaduck# git log -1 --stat
commit 6445ee79506fe8c68662dfd637d17c74f5f59334 (HEAD -> work)
Author: Codex <codex@openai.com>
Date: Thu May 22 14:03:34 2025 +0000
fix(server): remove sqlite fallback
scubaduck/server.py | 67 ++++++++--------------------------------------------
tests/test_server.py | 54 ++----------------------------------------
2 files changed, 12 insertions(+), 109 deletions(-)
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck#I need to gather citation references and identify the lines I modified in the patch.
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck# nl -ba scubaduck/server.py |
sed -n '70,110p'
70 con.execute(f'CREATE VIEW "{t}" AS SELECT * FROM db."{t}"')
71 else:
72 con = duckdb.connect(path)
73 return con
74
75
76 _REL_RE = re.compile(
77 r"([+-]?\d+(?:\.\d*)?)\s*(hour|hours|day|days|week|weeks|fortnight|f
ortnights)",
78 re.IGNORECASE,
79 )
80
81
82 def parse_time(val: str | None) -> str | None:
83 """Parse an absolute or relative time string into ``YYYY-MM-DD HH:MM
:SS``."""
84 if val is None or val == "":
85 return None
86 s = val.strip()
87 if s.lower() == "now":
88 dt = datetime.now(timezone.utc)
89 return dt.replace(microsecond=0).strftime("%Y-%m-%d %H:%M:%S")
90
91 m = _REL_RE.fullmatch(s)
92 if m:
93 qty = float(m.group(1))
94 unit = m.group(2).lower()
95 delta: timedelta
96 if unit.startswith("hour"):
97 delta = timedelta(hours=qty)
98 elif unit.startswith("day"):
99 delta = timedelta(days=qty)
100 elif unit.startswith("week"):
101 delta = timedelta(weeks=qty)
102 elif unit.startswith("fortnight"):
103 delta = timedelta(weeks=2 * qty)
104 else: # pragma: no cover - defensive
105 raise ValueError(f"Unsupported unit: {unit}")
106 dt = datetime.now(timezone.utc) + delta
107 return dt.replace(microsecond=0).strftime("%Y-%m-%d %H:%M:%S")
108
109 dt = dtparser.parse(s)
110 return dt.replace(microsecond=0, tzinfo=None).strftime("%Y-%m-%d %H:
%M:%S")
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck#I’ll now capture lines around _load_database to include the new code changes.
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck# nl -ba scubaduck/server.py |
sed -n '50,74p'
50 if not path.exists():
51 raise FileNotFoundError(path)
52
53 ext = path.suffix.lower()
54 if ext == ".csv":
55 con = duckdb.connect()
56 con.execute(
57 f"CREATE TABLE events AS SELECT * FROM read_csv_auto('{path.
as_posix()}')"
58 )
59 elif ext in {".db", ".sqlite"}:
60 con = duckdb.connect()
61 con.execute("LOAD sqlite")
62 con.execute(f"ATTACH '{path.as_posix()}' AS db (TYPE SQLITE)")
63 tables = [
64 r[0]
65 for r in con.execute(
66 "SELECT name FROM sqlite_master WHERE type='table'"
67 ).fetchall()
68 ]
69 for t in tables:
70 con.execute(f'CREATE VIEW "{t}" AS SELECT * FROM db."{t}"')
71 else:
72 con = duckdb.connect(path)
73 return con
74
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck#I'll grab the snippet from the tests to show the updated approach.
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck# nl -ba tests/test_server.py
| sed -n '210,275p'
210 sqlite_file = tmp_path / "events.sqlite"
211 import sqlite3
212
213 conn = sqlite3.connect(sqlite_file)
214 conn.execute(
215 "CREATE TABLE events (timestamp TEXT, url LONGVARCHAR, title VAR
CHAR(10))"
216 )
217 conn.execute(
218 "INSERT INTO events VALUES ('2024-01-01 00:00:00', 'https://a.co
m', 'Home')"
219 )
220 conn.commit()
221 conn.close() # pyright: ignore[reportUnknownMemberType, reportAttri
buteAccessIssue]
222
223 app = server.create_app(sqlite_file)
224 client = app.test_client()
225 payload = {
226 "table": "events",
227 "start": "2024-01-01 00:00:00",
228 "end": "2024-01-01 01:00:00",
229 "order_by": "timestamp",
230 "columns": ["timestamp", "url", "title"],
231 }
232 rv = client.post(
233 "/api/query", data=json.dumps(payload), content_type="applicatio
n/json"
234 )
235 data = rv.get_json()
236 assert rv.status_code == 200
237 assert data["rows"][0][1] == "https://a.com"
238
239
240 def test_sqlite_bigint(tmp_path: Path) -> None:
241 sqlite_file = tmp_path / "big.sqlite"
242 import sqlite3
243
244 conn = sqlite3.connect(sqlite_file)
245 conn.execute("CREATE TABLE events (timestamp TEXT, value INTEGER)")
246 big_value = 13385262862605259
247 conn.execute(
248 "INSERT INTO events VALUES ('2024-01-01 00:00:00', ?)",
249 (big_value,),
250 )
251 conn.commit()
252 conn.close() # pyright: ignore[reportUnknownMemberType, reportAttri
buteAccessIssue]
253
254 app = server.create_app(sqlite_file)
255 client = app.test_client()
256 payload = {
257 "table": "events",
258 "order_by": "timestamp",
259 "columns": ["timestamp", "value"],
260 }
261 rv = client.post(
262 "/api/query", data=json.dumps(payload), content_type="applicatio
n/json"
263 )
264 data = rv.get_json()
265 assert rv.status_code == 200
266 assert data["rows"][0][1] == big_value
267
268
269 def test_integer_time_column(tmp_path: Path) -> None:
270 csv_file = tmp_path / "events.csv"
271 csv_file.write_text("created,event\n1704067200,login\n1704070800,log
out\n")
272 app = server.create_app(csv_file)
273 client = app.test_client()
274 payload = {
275 "table": "events",
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck#I'll check surrounding lines to reference the missing function patch in the final summary.
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck# nl -ba scubaduck/server.py |
sed -n '30,69p'
30 end: str | None = None
31 order_by: str | None = None
32 order_dir: str = "ASC"
33 limit: int | None = None
34 columns: list[str] = field(default_factory=lambda: [])
35 filters: list[Filter] = field(default_factory=lambda: [])
36 derived_columns: dict[str, str] = field(default_factory=lambda: {})
37 graph_type: str = "samples"
38 group_by: list[str] = field(default_factory=lambda: [])
39 aggregate: str | None = None
40 show_hits: bool = False
41 x_axis: str | None = None
42 granularity: str = "Auto"
43 fill: str = "0"
44 table: str = "events"
45 time_column: str = "timestamp"
46 time_unit: str = "s"
47
48
49 def _load_database(path: Path) -> duckdb.DuckDBPyConnection:
50 if not path.exists():
51 raise FileNotFoundError(path)
52
53 ext = path.suffix.lower()
54 if ext == ".csv":
55 con = duckdb.connect()
56 con.execute(
57 f"CREATE TABLE events AS SELECT * FROM read_csv_auto('{path.
as_posix()}')"
58 )
59 elif ext in {".db", ".sqlite"}:
60 con = duckdb.connect()
61 con.execute("LOAD sqlite")
62 con.execute(f"ATTACH '{path.as_posix()}' AS db (TYPE SQLITE)")
63 tables = [
64 r[0]
65 for r in con.execute(
66 "SELECT name FROM sqlite_master WHERE type='table'"
67 ).fetchall()
68 ]
69 for t in tables:
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck#I’m focusing on lines around 68 and further to capture relevant updates.
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck# nl -ba scubaduck/server.py |
sed -n '68,76p'
68 ]
69 for t in tables:
70 con.execute(f'CREATE VIEW "{t}" AS SELECT * FROM db."{t}"')
71 else:
72 con = duckdb.connect(path)
73 return con
74
75
76 _REL_RE = re.compile(
(scubaduck) root@3a541e64f0a7:/workspace/scubaduck#I'll summarize the _load_database function and create a PR message with root instructions.