diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml new file mode 100644 index 0000000..729134c --- /dev/null +++ b/.github/workflows/build.yaml @@ -0,0 +1,278 @@ +--- +name: Build and Test + +permissions: + contents: read + +on: + push: + branches: + - main + pull_request: + merge_group: + +# Supersede in-flight runs on the same ref. Never cancel in a merge queue: a +# cancelled merge_group run reports failure and evicts the PR from the queue. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +env: + CARGO_TERM_COLOR: always + RUST_TOOLCHAIN_VERSION: "1.95.0" + +# Every job below names a runner image rather than a `-latest` alias, so an +# image roll cannot change what a merge is gated on. The label cannot be lifted +# into a variable: `runs-on` accepts no `env` context, and the one context that +# would work, `vars`, holds its value in repository settings rather than here. +jobs: + # The whole gate: `cargo test`, formatting, clippy (which is what enforces the + # unwrap_used / unwrap_in_result / panic denies from Cargo.toml), rustdoc, + # cargo-deny, cargo-sort and shellcheck. + # + # This lives here rather than in its own workflow because `needs:` cannot + # cross workflows, and a lint gate the required check does not observe is not + # a gate. It runs the hooks rather than the underlying commands so that CI and + # `pre-commit run --all-files` cannot drift apart -- CLAUDE.md points + # contributors at that command as the single source of truth for what must + # pass, which is only true if CI runs the same thing. + pre-commit: + name: pre-commit + runs-on: ubuntu-24.04 + timeout-minutes: 20 + steps: + # The cargo-test pre-commit hook links libodbc via odbc-sys. + - name: Install host dependencies + uses: awalsh128/cache-apt-pkgs-action@acb598e5ddbc6f68a970c5da0688d2f3a9f04d05 # v1.6.0 + with: + packages: unixodbc-dev + # A cache key, not a runner label, but it tracks the runner image so + # that bumping the image invalidates the cached .deb files. + version: ubuntu-24.04 + + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + persist-credentials: false + + - name: Install Rust ${{ env.RUST_TOOLCHAIN_VERSION }} toolchain + uses: dtolnay/rust-toolchain@b3b07ba8b418998c39fb20f53e8b695cdcc8de1b # 1.95.0 + with: + toolchain: ${{ env.RUST_TOOLCHAIN_VERSION }} + components: rustfmt, clippy + + - name: Setup Rust Cache + uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 + + - name: Install cargo-deny and cargo-sort + uses: taiki-e/install-action@97a5807a604e12de3a13b52d868ebecaeeea757c # v2.75.4 + with: + tool: cargo-deny,cargo-sort + + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1 + + # This suite needs only unixODBC and the sqlite3 CLI, no server and no + # container, so it runs on a standard runner in seconds and is worth + # gating every pull request on. The Trino driver cannot do this, which is + # part of why this driver exists. + sqlite-integration: + name: SQLite Integration Tests + runs-on: ubuntu-24.04 + timeout-minutes: 20 + needs: [pre-commit] + steps: + - name: Install host dependencies + uses: awalsh128/cache-apt-pkgs-action@acb598e5ddbc6f68a970c5da0688d2f3a9f04d05 # v1.6.0 + with: + packages: unixodbc-dev unixodbc sqlite3 + version: ubuntu-24.04 + + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + persist-credentials: false + + - name: Install Rust ${{ env.RUST_TOOLCHAIN_VERSION }} toolchain + uses: dtolnay/rust-toolchain@b3b07ba8b418998c39fb20f53e8b695cdcc8de1b + with: + toolchain: ${{ env.RUST_TOOLCHAIN_VERSION }} + + - name: Setup Rust Cache + uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 + + - name: Install uv + uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57 # v8.0.0 + + - name: Run SQLite integration tests + run: | + ./integration-tests/setup.sh + # --skip-cargo-test: the pre-commit job above already ran it. + ./integration-tests/run-tests.sh --skip-cargo-test + + unit-tests-windows: + name: Unit Tests (Windows) + runs-on: windows-2022 + timeout-minutes: 30 + steps: + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + persist-credentials: false + + - name: Install Rust ${{ env.RUST_TOOLCHAIN_VERSION }} toolchain + uses: dtolnay/rust-toolchain@b3b07ba8b418998c39fb20f53e8b695cdcc8de1b + with: + toolchain: ${{ env.RUST_TOOLCHAIN_VERSION }} + targets: x86_64-pc-windows-gnu + + # A separate cache key: the Linux job's artefacts are a different target + # triple and sharing the key would thrash both. + - name: Setup Rust Cache + uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 + with: + key: windows-gnu-test + + # The GNU target's linker, and the C compiler libsqlite3-sys needs to + # build the bundled SQLite amalgamation. The runner image ships MSYS2, + # but its mingw64 bin directory is not on PATH by default. + - name: Add MinGW to PATH + run: echo "C:\msys64\mingw64\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append + + # `--target x86_64-pc-windows-gnu`, not the runner's default MSVC triple. + # That is the target release.yaml builds and packaging/build-archives.sh + # ships, and a suite passing against a toolchain nobody receives is only + # evidence about that toolchain. odbc-sys links odbc32, which comes with + # the Windows SDK already on the runner, so there is no equivalent of the + # unixodbc-dev install the Linux jobs need. + # + # This is also the only job that compiles `backend::setup`'s + # `#[cfg(windows)]` module, which is the half of the setup dialog that + # calls into kernel32. + - name: Run unit tests + run: cargo test --locked --target x86_64-pc-windows-gnu + + # Builds both shipping artifacts the way release.yaml does, and checks the two + # properties that are invisible in a unit test run: that the DLL exports the + # ODBC entry points, and that what each artifact links at load time still + # matches packaging/sbom-native.json. The SBOM declares native dependencies by + # hand, since no cargo metadata describes them, so nothing but this check keeps + # the declaration true. + release-artifacts: + name: Release Artifacts + runs-on: ubuntu-24.04 + timeout-minutes: 20 + needs: [pre-commit] + steps: + - name: Install MinGW cross-compiler + run: sudo apt-get update && sudo apt-get install -y gcc-mingw-w64-x86-64 unixodbc-dev + + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + persist-credentials: false + + - name: Install Rust ${{ env.RUST_TOOLCHAIN_VERSION }} toolchain + uses: dtolnay/rust-toolchain@b3b07ba8b418998c39fb20f53e8b695cdcc8de1b + with: + toolchain: ${{ env.RUST_TOOLCHAIN_VERSION }} + targets: x86_64-pc-windows-gnu + + - name: Setup Rust Cache + uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 + with: + key: windows-gnu + + - name: Build Linux shared library + run: cargo build --locked --release + + - name: Build Windows DLL + run: cargo build --locked --target x86_64-pc-windows-gnu --release + + # Graded on what the DLL actually exports, not on the last command in a + # pipeline. The previous form piped a symbol count into `xargs echo`, and + # the step's exit status was `xargs`'s, which is 0 whatever it echoes: a + # DLL exporting nothing at all printed "0 ODBC symbols exported" and + # passed. The named set catches the entry points an application reaches + # the driver through, and the floor catches a wholesale regression in + # core's `forward_ffi!` even if these particular names survive. + - name: Verify DLL exports + run: | + DLL=target/x86_64-pc-windows-gnu/release/stackable_odbc_sqlite.dll + EXPORTS=$(x86_64-w64-mingw32-objdump -p "$DLL" \ + | awk '/Export Address Table/,/Ordinal base/' \ + | grep -oE '\b(SQL|Config)[A-Za-z]+\b' | sort -u) + echo "$EXPORTS" | tr '\n' ' '; echo + + missing="" + for sym in SQLAllocHandle SQLFreeHandle SQLDriverConnectW SQLConnectW \ + SQLBrowseConnectW SQLDisconnect SQLPrepareW SQLExecute \ + SQLExecDirectW SQLBindParameter SQLDescribeParam SQLFetch \ + SQLGetData SQLNumResultCols SQLDescribeColW SQLGetInfoW \ + SQLGetTypeInfoW SQLGetDiagRecW SQLTablesW SQLColumnsW \ + SQLEndTran SQLCancel ConfigDSNW; do + grep -qx "$sym" <<< "$EXPORTS" || missing="$missing $sym" + done + if [ -n "$missing" ]; then + echo "::error::the DLL does not export:$missing" + exit 1 + fi + + # The DLL exports 61 and the Linux .so 60, the difference being + # ConfigDSNW, which is `#[cfg(windows)]`. + count=$(echo "$EXPORTS" | grep -c .) + if [ "$count" -lt 55 ]; then + echo "::error::only $count ODBC symbols exported; expected at least 55" + exit 1 + fi + echo "SQLite DLL: $count ODBC symbols exported, all required names present" + + # build.rs embeds this, and it is what stops the ODBC Data Source + # Administrator listing the driver as "Not marked". A cross-build with no + # windres on PATH fails loudly, but a change to build.rs that silently + # stops emitting the resource would not, so the section is asserted here. + - name: Verify the DLL carries a version resource + run: | + x86_64-w64-mingw32-objdump -h target/x86_64-pc-windows-gnu/release/stackable_odbc_sqlite.dll \ + | grep -q '\.rsrc' || { echo "::error::the DLL carries no .rsrc section"; exit 1; } + echo "Version resource present." + + # Two different assertions behind one flag. For the .so it compares + # DT_NEEDED against the sonames sbom-native.json declares, in both + # directions. For the .dll it asserts the mingw runtime is still linked + # statically: the release archive ships no runtime DLL, so an artifact + # that imported one would fail to load on a user's machine. + # + # Only the release binaries are checked, and only here rather than in + # release.yaml, because a pull request is where a dependency change can + # still be reverted cheaply. + - name: Verify declared native dependencies + run: | + ./packaging/sbom.sh --check-native target/release/libstackable_odbc_sqlite.so + ./packaging/sbom.sh --check-native target/x86_64-pc-windows-gnu/release/stackable_odbc_sqlite.dll + + # Single required check for branch protection rules. + finished: + name: Finished Build and Test + if: always() + needs: + - pre-commit + - sqlite-integration + - unit-tests-windows + - release-artifacts + runs-on: ubuntu-24.04 + timeout-minutes: 5 + steps: + # Derived from needs.* rather than a hand-written list of job names: a job + # added to `needs` above but forgotten here would otherwise be silently + # non-blocking. + - name: Check job results + env: + RESULTS: ${{ join(needs.*.result, ' ') }} + run: | + for result in $RESULTS; do + if [[ "$result" != "success" ]]; then + echo "One or more jobs did not succeed: $RESULTS" + exit 1 + fi + done + echo "All jobs passed: $RESULTS" diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 0000000..9ac04e4 --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,265 @@ +--- +name: Release + +on: + push: + tags: + # The tag format cargo-release produces; see release.toml. + - "v*" + workflow_dispatch: + +# Read at the top level; the two jobs that need more grant it to themselves. +# Attestation needs an OIDC token, and only the publishing job writes. +permissions: + contents: read + +env: + CARGO_TERM_COLOR: always + RUST_TOOLCHAIN_VERSION: "1.95.0" + # Both pinned rather than floating. packaging/test-sbom.sh asserts the shape of + # what syft emits and how cargo-auditable's .dep-v0 section reads, so a release + # of either that changes that shape has to be adopted deliberately and + # re-verified, not picked up silently on the next tag push. + SYFT_VERSION: "v1.50.0" + CARGO_AUDITABLE_VERSION: "0.7.5" + +# Every job below names a runner image rather than a `-latest` alias, so an +# image roll cannot change what a tagged release is built against. See +# build.yaml for why the label is repeated rather than named once. +jobs: + verify-version: + name: Verify tag matches Cargo.toml + runs-on: ubuntu-24.04 + outputs: + version: ${{ steps.extract.outputs.version }} + steps: + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + persist-credentials: false + + - id: extract + name: Compare tag and Cargo.toml version + run: | + TAG="${GITHUB_REF#refs/tags/}" + TAG_VERSION="${TAG#v}" + CARGO_VERSION=$(grep -m1 '^version = ' Cargo.toml | sed -E 's/version = "(.+)"/\1/') + if [ "$TAG_VERSION" != "$CARGO_VERSION" ]; then + echo "::error::Tag $TAG says version $TAG_VERSION but Cargo.toml has $CARGO_VERSION" + exit 1 + fi + echo "version=$CARGO_VERSION" >> "$GITHUB_OUTPUT" + echo "Verified: releasing stackable-odbc-sqlite $CARGO_VERSION" + + # The suite is server-free, so unlike the Trino driver's it can gate a + # release as well as a pull request. + integration-test: + name: SQLite Integration Tests + runs-on: ubuntu-24.04 + needs: [verify-version] + steps: + - name: Install host dependencies + uses: awalsh128/cache-apt-pkgs-action@acb598e5ddbc6f68a970c5da0688d2f3a9f04d05 # v1.6.0 + with: + packages: unixodbc-dev unixodbc sqlite3 + version: ubuntu-24.04 + + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + persist-credentials: false + + - name: Install Rust ${{ env.RUST_TOOLCHAIN_VERSION }} toolchain + uses: dtolnay/rust-toolchain@b3b07ba8b418998c39fb20f53e8b695cdcc8de1b + with: + toolchain: ${{ env.RUST_TOOLCHAIN_VERSION }} + + - name: Setup Rust Cache + uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 + + - name: Install uv + uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57 # v8.0.0 + + - name: Run SQLite integration tests + run: | + ./integration-tests/setup.sh + ./integration-tests/run-tests.sh + + build-and-package: + name: Build and package release archives + runs-on: ubuntu-24.04 + needs: [verify-version, integration-test] + # id-token and attestations are what actions/attest-* exchange for a + # Sigstore signing certificate; contents stays read, since this job + # publishes nothing. + permissions: + contents: read + id-token: write + attestations: write + steps: + - name: Install host dependencies + run: | + sudo apt-get update + sudo apt-get install -y unixodbc-dev gcc-mingw-w64-x86-64 zip + + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + persist-credentials: false + + - name: Install Rust ${{ env.RUST_TOOLCHAIN_VERSION }} toolchain + uses: dtolnay/rust-toolchain@b3b07ba8b418998c39fb20f53e8b695cdcc8de1b + with: + toolchain: ${{ env.RUST_TOOLCHAIN_VERSION }} + targets: x86_64-pc-windows-gnu + + - name: Setup Rust Cache + uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 + with: + key: release + + # packaging/sbom.sh reads the .dep-v0 section cargo-auditable embeds, and + # refuses an artifact without one. Both tools are therefore preconditions + # of packaging, not optional extras. + - name: Install cargo-auditable + uses: taiki-e/install-action@97a5807a604e12de3a13b52d868ebecaeeea757c # v2.75.4 + with: + tool: cargo-auditable@${{ env.CARGO_AUDITABLE_VERSION }} + + - name: Install syft + uses: anchore/sbom-action/download-syft@e22c389904149dbc22b58101806040fa8d37a610 # v0.24.0 + with: + syft-version: ${{ env.SYFT_VERSION }} + + # `cargo auditable build`, not `cargo build`: the plain form links the same + # code but embeds no dependency graph, and the SBOM then lists a handful of + # components instead of the whole tree. + # + # --locked so the released binary is built from the versions Cargo.lock + # pins. The SBOM describes what was linked, so an unlocked build would + # produce an accurate document about an unintended dependency set. + - name: Build Linux release binary + run: cargo auditable build --locked --release + + - name: Build Windows release binary (cross) + run: cargo auditable build --locked --release --target x86_64-pc-windows-gnu + + - name: Assemble release archives + env: + VERSION: ${{ needs.verify-version.outputs.version }} + run: ./packaging/build-archives.sh + + - name: Sanity-check archive contents + env: + VERSION: ${{ needs.verify-version.outputs.version }} + run: | + DIST=packaging/dist + LINUX="$DIST/stackable-odbc-sqlite-${VERSION}-linux-x64.tar.gz" + WINDOWS="$DIST/stackable-odbc-sqlite-${VERSION}-windows-x64.zip" + + echo "--- Linux archive ---" + tar -tzf "$LINUX" + for f in libstackable_odbc_sqlite.so libstackable_odbc_sqlite.so.cdx.json \ + install.sh uninstall.sh README.md LICENSE; do + tar -tzf "$LINUX" | grep -qx "./$f" || { echo "::error::missing $f in linux archive"; exit 1; } + done + + echo "--- Windows archive ---" + unzip -l "$WINDOWS" + # configure-dsn.ps1 is load-bearing rather than an extra: install.bat + # refuses to register the driver without it. + for f in stackable_odbc_sqlite.dll stackable_odbc_sqlite.dll.cdx.json \ + configure-dsn.ps1 install.bat uninstall.bat README.md LICENSE; do + unzip -l "$WINDOWS" | grep -q " $f\$" || { echo "::error::missing $f in windows archive"; exit 1; } + done + + echo "--- Checksums ---" + (cd "$DIST" && sha256sum -c sha256sums.txt) + + echo "Archive sanity check passed." + + # TODO(@maltesander): The published binaries are unsigned. Authenticode + # for stackable_odbc_sqlite.dll needs a code-signing + # certificate, which has not been bought. Until then + # Windows SmartScreen warns on the installer. The + # attestations below are a different guarantee: they + # prove where an artifact was built, not who vouches + # for it, and no operating system consults them. + + # Signs a statement that these files came out of this workflow, at this + # commit, and records it in the public transparency log. Verified with + # `gh attestation verify --repo stackabletech/stackable-odbc-sqlite`. + # sha256sums.txt is included so the SBOM assets, which it covers, are + # reachable from an attested file. + - name: Attest build provenance + uses: actions/attest-build-provenance@0f67c3f4856b2e3261c31976d6725780e5e4c373 # v4.1.1 + with: + subject-path: | + packaging/dist/*.tar.gz + packaging/dist/*.zip + packaging/dist/sha256sums.txt + + # One call per artifact, because each binds exactly one SBOM to one + # subject. The CycloneDX document is the one attested; the SPDX one beside + # it is a conversion of the same data for consumers that need that format. + - name: Attest SBOM for the Linux archive + uses: actions/attest-sbom@c604332985a26aa8cf1bdc465b92731239ec6b9e # v4.1.0 + with: + subject-path: packaging/dist/stackable-odbc-sqlite-${{ needs.verify-version.outputs.version }}-linux-x64.tar.gz + sbom-path: packaging/dist/stackable-odbc-sqlite-${{ needs.verify-version.outputs.version }}-linux-x64.cdx.json + + - name: Attest SBOM for the Windows archive + uses: actions/attest-sbom@c604332985a26aa8cf1bdc465b92731239ec6b9e # v4.1.0 + with: + subject-path: packaging/dist/stackable-odbc-sqlite-${{ needs.verify-version.outputs.version }}-windows-x64.zip + sbom-path: packaging/dist/stackable-odbc-sqlite-${{ needs.verify-version.outputs.version }}-windows-x64.cdx.json + + - name: Upload archives as workflow artifacts + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: release-archives + path: packaging/dist/* + retention-days: 7 + + publish-release: + name: Publish GitHub Release + runs-on: ubuntu-24.04 + needs: [verify-version, build-and-package] + # The only job that writes, and it writes exactly one thing: the release. + permissions: + contents: write + steps: + - name: Download archives + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: release-archives + path: dist + + - name: Determine prerelease flag + id: prerelease + env: + VERSION: ${{ needs.verify-version.outputs.version }} + run: | + if [[ "$VERSION" == *-* ]]; then + echo "flag=true" >> "$GITHUB_OUTPUT" + else + echo "flag=false" >> "$GITHUB_OUTPUT" + fi + + - name: Create GitHub Release + uses: softprops/action-gh-release@72f2c25fcb47643c292f7107632f7a47c1df5cd8 # v2.3.2 + with: + tag_name: ${{ github.ref_name }} + name: stackable-odbc-sqlite ${{ needs.verify-version.outputs.version }} + generate_release_notes: true + prerelease: ${{ steps.prerelease.outputs.flag }} + # Each archive already carries its own CycloneDX SBOM, so an offline + # install has one. The four standalone documents are here for whoever + # needs to read an SBOM without downloading and unpacking a release, + # and in SPDX as well as CycloneDX because tools take one or the other. + # sha256sums.txt covers every file listed above it. + files: | + dist/stackable-odbc-sqlite-${{ needs.verify-version.outputs.version }}-linux-x64.tar.gz + dist/stackable-odbc-sqlite-${{ needs.verify-version.outputs.version }}-windows-x64.zip + dist/stackable-odbc-sqlite-${{ needs.verify-version.outputs.version }}-linux-x64.cdx.json + dist/stackable-odbc-sqlite-${{ needs.verify-version.outputs.version }}-linux-x64.spdx.json + dist/stackable-odbc-sqlite-${{ needs.verify-version.outputs.version }}-windows-x64.cdx.json + dist/stackable-odbc-sqlite-${{ needs.verify-version.outputs.version }}-windows-x64.spdx.json + dist/sha256sums.txt diff --git a/.github/workflows/scorecard.yaml b/.github/workflows/scorecard.yaml new file mode 100644 index 0000000..01c3cca --- /dev/null +++ b/.github/workflows/scorecard.yaml @@ -0,0 +1,49 @@ +--- +name: OpenSSF Scorecard + +# Scorecard grades repository configuration rather than the crate. +# It scores pinned action SHAs, workflow permissions and release +# provenance, which makes it a regression check on the supply-chain work. +# +# `publish_results` and the SARIF upload both require a public repository. +# Runs before this one goes public are expected to fail. + +on: + branch_protection_rule: + schedule: + # Every Monday at 05:30 UTC: https://crontab.guru/#30_5_*_*_1 + - cron: '30 5 * * 1' + push: + branches: + - main + workflow_dispatch: + +permissions: read-all + +jobs: + analysis: + name: Scorecard analysis + runs-on: ubuntu-24.04 + permissions: + # Upload the results to the code-scanning dashboard. + security-events: write + # Publish results to the public Scorecard API, which backs the badge. + id-token: write + contents: read + actions: read + steps: + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + persist-credentials: false + + - name: Run analysis + uses: ossf/scorecard-action@2d1146689b8cda280b9bc96326124645441f03bc # v2.4.4 + with: + results_file: results.sarif + results_format: sarif + publish_results: true + + - name: Upload results to code scanning + uses: github/codeql-action/upload-sarif@a2983b8bed1923f44751c5c43237f479442827b3 # v3.37.4 + with: + sarif_file: results.sarif diff --git a/.github/workflows/security_audit.yaml b/.github/workflows/security_audit.yaml new file mode 100644 index 0000000..ef7afc1 --- /dev/null +++ b/.github/workflows/security_audit.yaml @@ -0,0 +1,25 @@ +--- +name: Daily Security Audit + +on: + schedule: + # Run every day at 04:15 UTC: https://crontab.guru/#15_4_*_*_* + - cron: '15 4 * * *' + workflow_dispatch: + +permissions: + contents: read + +jobs: + audit: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + persist-credentials: false + + # This `token` is the action's own input, not checkout's: audit-check + # needs it to post the advisory annotations onto the run. + - uses: rustsec/audit-check@69366f33c96575abad1ee0dba8212993eecbe998 # v2.0.0 + with: + token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..caff3c4 --- /dev/null +++ b/.gitignore @@ -0,0 +1,28 @@ +debug/ +target/ +**/*.rs.bk +.worktrees/ + +.idea/ +*.iws +*.iml +.vscode/ + +# Generated via ctags -R. +tags + +# Local agent working notes (SDD reports); never part of the shipped tree +.superpowers/ + +# Release packaging output +packaging/dist/ + +# integration-tests/generated/ has its own .gitignore; the ODBC config setup.sh +# writes there embeds absolute paths. + +# Python bytecode from the test scripts +__pycache__/ +*.pyc + +# Local cargo overrides, e.g. a [patch] pointing core at a sibling checkout. +.cargo/ diff --git a/.markdownlint.yaml b/.markdownlint.yaml new file mode 100644 index 0000000..783004c --- /dev/null +++ b/.markdownlint.yaml @@ -0,0 +1,28 @@ +--- +# All defaults or options can be checked here: +# https://github.com/DavidAnson/markdownlint/blob/main/schema/.markdownlint.yaml + +# Default state for all rules +default: true + +# MD013/line-length - Line length +MD013: + # Number of characters + line_length: 9999 + # Number of characters for headings + heading_line_length: 9999 + # Number of characters for code blocks + code_block_line_length: 9999 + +# MD024/no-duplicate-heading/no-duplicate-header - Multiple headings with the same content +MD024: + # Only check sibling headings + siblings_only: true + +# MD040/fenced-code-language - Fenced code blocks should have a language specified +# We use plain fenced blocks for ODBC config files and output examples +MD040: false + +# MD060/table-column-style - Table column alignment +# Too strict for our tables +MD060: false diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..947dfdd --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,96 @@ +--- +default_language_version: + node: system + +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: cef0300fd0fc4d2a87a85fa2093c6b283ea36f4b # 5.0.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: check-toml + - id: check-merge-conflict + - id: mixed-line-ending + - id: detect-aws-credentials + args: ["--allow-missing-credentials"] + - id: detect-private-key + + - repo: https://github.com/igorshubovych/markdownlint-cli + rev: 192ad822316c3a22fb3d3cc8aa6eafa0b8488360 # 0.45.0 + hooks: + - id: markdownlint + + - repo: https://github.com/koalaman/shellcheck-precommit + rev: 2491238703a5d3415bb2b7ff11388bf775372f29 # 0.10.0 + hooks: + - id: shellcheck + # -x follows `source`d files. The integration-test scripts share + # lib.sh, and without it every one of them reports SC1091 for a file + # that is right there and checkable. + args: ["--severity=info", "-x"] + + - repo: local + hooks: + # The packaging files are in `files:` because tests in src/lib.rs + # `include_str!` them: the Windows dialog is checked against the + # connection-string parser, and sbom-native.json's SQLite version against + # the library that is actually linked. Changing one of those without + # touching any .rs file is exactly the case those tests exist to catch, + # and without this the hook would not run for it. + - id: cargo-test + name: cargo-test + language: system + entry: cargo test --locked + stages: [pre-commit, pre-merge-commit] + pass_filenames: false + files: \.rs$|Cargo\.(toml|lock)|^packaging/(sbom-native\.json|windows/(configure-dsn\.ps1|install\.bat))$ + + - id: cargo-rustfmt + name: cargo-rustfmt + language: system + entry: cargo fmt --all -- --check + stages: [pre-commit, pre-merge-commit] + pass_filenames: false + files: \.rs$ + + - id: cargo-clippy + name: cargo-clippy + language: system + entry: cargo clippy --locked --all-targets -- -D warnings + stages: [pre-commit, pre-merge-commit] + pass_filenames: false + files: \.rs$ + + # Broken intra-doc links are warnings, not errors, so they reach a + # published doc build silently. -D warnings promotes them. Runs in well + # under a second because the dependency graph is already built above. + # + # --document-private-items, because almost everything in this crate is + # private -- `backend::setup` entirely so -- and without it their doc + # comments are checked by nothing. Not `--all-targets`: that builds the + # test target, sets `cfg(test)`, and would pull the `#[cfg(test)]` + # modules into the check. + - id: cargo-doc + name: cargo-doc + language: system + entry: env RUSTDOCFLAGS=-Dwarnings cargo doc --locked --no-deps --document-private-items + stages: [pre-commit, pre-merge-commit] + pass_filenames: false + files: \.rs$|Cargo\.(toml|lock) + + - id: cargo-sort + name: cargo-sort + language: system + entry: cargo sort --grouped --check + stages: [pre-commit, pre-merge-commit] + pass_filenames: false + files: Cargo\.toml$ + + - id: cargo-deny + name: cargo-deny + language: system + entry: cargo deny --locked check + stages: [pre-commit, pre-merge-commit] + pass_filenames: false + files: Cargo\.(toml|lock)|deny\.toml diff --git a/.readme/static/borrowed/Icon_Stackable.svg b/.readme/static/borrowed/Icon_Stackable.svg new file mode 100644 index 0000000..35e132a --- /dev/null +++ b/.readme/static/borrowed/Icon_Stackable.svg @@ -0,0 +1,20 @@ + + + + + + + + + + + + + + + + + + + + diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..1fd0d78 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,692 @@ +# Agent Guide + +Implementation details for AI agents working on `stackable-odbc-sqlite`. + +This crate is an ODBC driver for [SQLite](https://sqlite.org). It contains +**only** SQLite-specific code: the `Backend` and `StatementBackend` +implementations, connection-string parsing, SQLite-to-ODBC type conversion, ODBC +escape-sequence translation, and the catalog and metadata functions. Everything +generic (handle management, UTF-16 marshalling, diagnostics, panic safety, and +the C ABI entry points) lives in +[`stackable-odbc-core`](https://github.com/stackabletech/stackable-odbc-core). + +## Quick Reference + +| Topic | When to Read | +|-------|-------------| +| [Architecture](#architecture-of-this-crate) | Finding the module a change belongs in | +| [Relationship to core](#relationship-to-stackable-odbc-core) | Deciding whether a change belongs here at all | +| [Conventions](#conventions) | Any code change | +| [Backend error mapping](#backend-error-mapping) | Touching an error path | +| [Declaring capabilities](#declaring-capabilities) | Adding or changing any `SQLGetInfo` value | +| [Transactions](#transactions) | Touching `SQLEndTran`, autocommit or cursor behaviour | +| [Cancellation](#cancellation) | Touching `SQLCancel` or `SQL_ATTR_QUERY_TIMEOUT` | +| [`row_count` has three answers](#row_count-has-three-answers-not-two) | Touching `SQLRowCount` or the execute path | +| [Catalog functions](#catalog-functions) | Touching anything in `metadata.rs` | +| [Connection string keys](#connection-string-keys) | Adding or changing a parameter | +| [Testing](#testing) | Writing or running tests | +| [Packaging and release](#packaging-and-release) | Cutting a release | + +```bash +cargo build # needs unixodbc-dev +cargo test # unit + FFI tests; needs no server +cargo clippy --all-targets -- -D warnings +pre-commit run --all-files # the gate; run before every commit + +./integration-tests/setup.sh # build driver, create the DB, write ODBC config +./integration-tests/run-tests.sh # run the integration suite +``` + +## Architecture of this crate + +| Path | Responsibility | +|------|----------------| +| `src/lib.rs` | The `forward_ffi!` invocation, the crate docs, and the packaging consistency tests | +| `src/backend.rs` | `SqliteBackend`, `SqliteConnection`, `SqliteStatement`, `SqliteError`, `map_sqlite_error` | +| `src/backend/execute.rs` | `exec_direct`, `prepare`, `execute`, and the `StatementBackend` impl | +| `src/backend/info.rs` | `SQLGetInfo` answers and the capability bitmaps, plus the snapshot test | +| `src/backend/metadata.rs` | The catalog row producers: tables, columns, primary keys, statistics, special columns | +| `src/backend/setup.rs` | `Backend::configure_dsn`, the Windows DSN setup dialog | +| `src/backend/params.rs` | Deliberately empty. Parameter binding is inline in `execute.rs`; the entry points are core's | +| `src/backend/types/connect_params.rs` | `SqliteConnectParams` | +| `src/escape_dialect.rs` | ODBC escape-sequence translation for SQLite's dialect | +| `src/type_conversion.rs` | SQLite storage classes and declared types → ODBC SQL types | +| `src/ffi_integration_tests.rs` | Tests that drive the real C ABI entry points | +| `build.rs` | Embeds the Windows version resource with `windres` | +| `benches/fetch_sqlite.rs` | Criterion fetch-throughput benchmark through the full FFI path | + +### Result sets are materialised eagerly + +`SqliteStatement` holds `rows: Vec>` and `cursor: i64`, an +index into an in-memory snapshot rather than a live SQLite cursor. +`exec_direct` collects every row before returning, and the +`rusqlite::Statement` is finalized at that point. + +This is load-bearing well beyond memory use. It is why the cursor-behaviour +hooks report `Preserve`, why `SQLEndTran` cannot disturb a cursor, and why +concurrency is a non-issue. Changing it is not a local optimisation. See +[Transactions](#transactions). + +## Relationship to stackable-odbc-core + +Core is a git dependency, pinned in `Cargo.toml`, and cargo fetches it for you. +To build against a local checkout instead, add a `[patch]` to your own +`.cargo/config.toml`; see +[CONTRIBUTING.md](CONTRIBUTING.md#working-on-core-at-the-same-time) for the +mechanics and the `Cargo.lock` caveat. This crate is not published to +crates.io. Releases are GitHub Release archives built by +`.github/workflows/release.yaml`. + +| Concern | Owner | +|---------|-------| +| Handle allocation, tag validation, `panic_safe` | core | +| UTF-16 marshalling, diagnostics, `SQLGetDiagRec` | core | +| The exported C ABI entry points (`forward_ffi!`): 60 `SQL*` functions, plus `ConfigDSNW` on Windows | core | +| `SQLGetInfo` marshalling and shape checking, cursor-state tracking | core | +| Every `SQLGetInfo` value that describes SQLite | this crate, see [Declaring capabilities](#declaring-capabilities) | +| `Backend` / `StatementBackend` trait definitions | core | +| Opening the database, executing, fetching | this crate | +| SQLite storage class → SQL type mapping, value conversion | this crate | +| Querying SQLite for catalog metadata | this crate, see [Catalog functions](#catalog-functions) | +| Catalog column layout, sort order, the `SQL_ALL_*` enumerations | core | +| Connection-string parsing | this crate | +| ODBC escape-sequence translation | this crate | +| All of `ConfigDSN`, apart from the dialog itself | core, see [The Windows setup dialog](#the-windows-setup-dialog) | + +`src/lib.rs` is the whole export surface: + +```rust +stackable_odbc_core::forward_ffi!(crate::backend::SqliteBackend); +``` + +That one line expands to every `#[unsafe(no_mangle)] pub unsafe extern "system"` +entry point. If a new ODBC function needs to be exported, it is added to core's +`forward_ffi!` macro, not here; this crate only implements whatever new trait +method it calls. + +## Conventions + +### Changelog + +Every change an application can observe (a reported `SQLGetInfo` value, a +SQLSTATE, a type mapping) gets an entry in `CHANGELOG.md` under +`## [Unreleased]`, following [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). +Internal refactoring does not. + +### Logging in backend methods + +Use `tracing` macros, never `println!`. The FFI entry points in core already log +their own arguments and return codes, so a backend method should log what core +cannot see: the SQL it is about to run, the SQLite error it just mapped, the +number of rows it materialised. `ODBC_LOG_LEVEL` / `ODBC_LOG_FILE` control +output; both are initialised by core. + +### Named constants + +ODBC attribute values, function IDs, and bitmap constants must use named `const` +definitions. Never write raw integer literals for ODBC-spec-defined values. Name +them after the ODBC spec name (e.g. `SQL_AUTOCOMMIT_ON`, `SQL_CB_PRESERVE`). + +**This applies to tests too.** Test code is where raw literals creep back in +most easily, usually with the spec name relegated to a trailing comment. A +comment is not a constant: + +```rust +// BAD: the value is unchecked and the name is only a comment +sql_bind_parameter::(stmt, 1, 1 /* SQL_PARAM_INPUT */, ..., -5 /* SQL_BIGINT */, ...); + +// GOOD: the compiler validates both +sql_bind_parameter::(stmt, 1, ParamType::Input as i16, ..., SqlDataType::EXT_BIG_INT.0, ...); +``` + +Prefer the `odbc-sys` type over defining a new constant when one exists. Most +spec values are already modelled: + +| Value | Use | +|-------|-----| +| `SQL_PARAM_INPUT`, `SQL_PARAM_OUTPUT`, … | `ParamType::Input as i16` | +| `SQL_BIGINT`, `SQL_VARCHAR`, `SQL_INTEGER`, … | `SqlDataType::EXT_BIG_INT.0` (note the `.0`) | +| `SQL_C_SBIGINT`, `SQL_C_WCHAR`, … | `CDataType::SBigInt as i16` | +| `SQL_ATTR_*` | `StatementAttribute::*` / `ConnectionAttribute::*` | +| `SQL_HANDLE_*` | `HandleType::*` | + +All are re-exported from `stackable_odbc_core::types`. **This crate takes no +direct `odbc-sys` dependency**, reaching those types only through core's +re-exports. Do not add `odbc-sys` to `Cargo.toml`. + +Core also re-exports the crate wholesale as `stackable_odbc_core::odbc_sys`, +so a type with no `types` re-export of its own is still reachable without a +direct dependency. Reach for that rather than hand-rolling a `#[repr(C)]` +mirror of a struct like `SQL_TIMESTAMP_STRUCT`: a mirror that drifts from the +real struct is two different types to the compiler and one silent ABI mismatch +to the application. + +### Type cast safety + +Never `as`-cast a value that can exceed the target type. Use `try_into()` and +map the failure to a SQLSTATE, or clamp deliberately with a comment saying why +the clamp is correct. Row counts, column sizes and buffer lengths all cross +between `usize`, `i64`, `u16` and `i16` in this crate. + +### Backend error mapping + +**Route every `rusqlite` error through `map_sqlite_error`** (`src/backend.rs`). +Never hand-build a `SqliteError` or `OdbcError` from a `rusqlite::Error` at the +call site; that function is the single place that decides the SQLSTATE. + +`map_sqlite_error` keeps the `rusqlite::Error` it classified in the variant's +`cause` field, and `From for OdbcError` turns that into +`with_native_error` (SQLite's *extended* result code, which is what separates +`SQLITE_CONSTRAINT_NOTNULL` from `SQLITE_CONSTRAINT_FOREIGNKEY`, as the SQLSTATE +cannot) and `with_source` (the causal chain). A new classified variant must +carry `cause` too, or it silently reports native code `0`. + +**One error type, both directions.** Every `Backend` and `StatementBackend` +method returns `Result<_, SqliteError>`, because core requires +`Into + From + Error + Send + Sync + 'static`. The +`From` direction is what lets a defaulted trait body construct an +error and still name `Self::Error`, and `SqliteError::Odbc` is where such an +error lands. Return `OdbcError::NoResultSet` and friends through `.into()` +rather than reclassifying them: the round trip is lossless, and reclassifying +would discard the SQLSTATE core chose. + +Convert raw integers to typed enums at the boundary with the `xxx_from_raw()` +functions from core, never `transmute`. + +### 08001 versus 08S01 + +`08001` ("client unable to establish connection") is only valid from the +connection functions. Once a connection exists, a failing link is `08S01` +("communication link failure"). That is the code the diagnostics tables of +`SQLExecute`, `SQLFetch`, `SQLGetInfo` and the rest actually list. + +For this driver `connect` is where real I/O happens: +`rusqlite::Connection::open` touches the filesystem, so a missing or unreadable +database file is `08001`. Failures after that point are `08S01`. + +## ODBC behaviour and design rationale + +Everything in this section describes behaviour an application can observe, and +each decision is anchored to a spec page or to measured SQLite behaviour. Read +the relevant part before changing what the driver reports. + +### Declaring capabilities + +Most of `Backend` is **required** methods that state what SQLite can +do: `alter_table_support`, `outer_join_capabilities`, `subqueries`, +`sql_conformance`, `supports_catalogs`, `identifier_case`, +`quoted_identifier_case`, `txn_capable`, `txn_isolation_options`, `integrity`, +`multiple_active_txn`, `special_characters`, `accessible_procedures`, +`dbms_name`, `dbms_version`, `table_types` and the rest. They are required, +with no default, deliberately: a defaulted capability is a claim no backend +ever made. `table_types` is required for the same reason and one of its own, +since an empty table-type list is an *answer* ("this data source has no table +types") rather than "unknown", and unlike catalogs and schemas there is no +`supports_*` method for core to derive it from. `special_characters` follows +the same principle, because `""` asserts that nothing beyond the alphanumerics +and underscore is legal unquoted, which is a claim rather than an absence. + +They all take `&Self::Connection`, because `SQLGetInfo` is a per-connection +call and a data source's capabilities can differ by server. Every one this +driver declares is a property of the SQLite `rusqlite` links, not of the file +opened, so each ignores the argument, but the answer must still be read +through a connection, and the tests do that via `info::tests::test_connection` +rather than calling the hook as a free function. `cursor_commit_behavior`, +`cursor_rollback_behavior`, `driver_name` and `driver_version` are the +exceptions and take none: `SQLGetInfo` must answer the cursor-behaviour pair +before a connection exists, and the Windows Driver Manager asks for driver +identity before `SQLDriverConnectW`. Note the split within the identity +group: `driver_name`/`driver_version` describe the driver and take no +connection, while `dbms_name`/`dbms_version` describe what was connected to and +take one. (`catalog_result_column_widths` is core's, defaulted here.) + +The same split runs through `get_info`. `sqlite_get_info` takes +`Option<&SqliteConnection>` (`None` on the pre-connect path) and hands it to +`default_get_info` / `common_get_info_raw`, which answer only what is knowable +without a data source and leave the rest. An arm that consults a capability +hook must therefore be guarded on the connection being present, which is why +`SQL_MAX_CATALOG_NAME_LEN` and `SQL_MAX_SCHEMA_NAME_LEN` only report `0` once +one is open. + +Four rules: + +**Declare it once.** A capability with a hook is answered *only* through the +hook, never also in `get_info_raw`. Core derives the info type from the hook, +so a second answer is a value that can disagree with itself, and the one an +application sees depends on which core consults first. `SQL_GETDATA_EXTENSIONS` +belongs to core for a related reason: it describes core's own fetch path rather +than any fact about SQLite. The snapshot test (`get_info_snapshot`) pins the +value an application sees regardless of who answers it, which is what makes +moving an answer safe. + +**Probe the bundled library, never the documentation or the system CLI.** +`rusqlite` links its own SQLite (3.53.2 via the `bundled` feature), and the +`sqlite3` binary on a developer's machine is a different version. The +difference is not academic: 3.51.3 rejects `ADD CONSTRAINT` and +`DROP CONSTRAINT` where 3.53.2 accepts them, so an `ALTER TABLE` bitmap written +from the system CLI's behaviour understates what the driver actually links. +`alter_table_capabilities_are_each_live_probed`, +`outer_join_capabilities_are_each_live_probed` and +`subqueries_are_each_live_probed` all execute the syntax they describe. + +**Probe the bits you do not claim, too.** A test that only checks what a bitmap +claims can overclaim forever, and a bitmap that only grows when someone notices +can understate forever. The negative half of the `ALTER TABLE` probe is what +catches a version difference like the one above. `SQL_KEYWORDS` goes further +and reads the list out of the library through `sqlite3_keyword_count` / +`sqlite3_keyword_name`, so it needs no maintenance at all. + +**Values must agree with each other.** The commonest defect in a capability +table is one fact stated twice, in opposite directions. These pairs each +describe the same thing and must be changed together: + +| Info type | Must agree with | +|---|---| +| `SQL_CATALOG_NAME` | `SQL_CATALOG_TERM`, `SQL_CATALOG_LOCATION`, `SQL_CATALOG_USAGE`, `SQL_CATALOG_NAME_SEPARATOR` | +| `SQL_OUTER_JOINS` | `SQL_OUTER_JOIN_CAPABILITIES` | +| `SQL_SQL_CONFORMANCE` | `SQL_GROUP_BY`, `SQL_CONCAT_NULL_BEHAVIOR`, `SQL_NON_NULLABLE_COLUMNS` | +| `SQL_SQL92_PREDICATES` (`SQL_SP_QUANTIFIED_COMPARISON`) | `SQL_SUBQUERIES` (`SQL_SQ_QUANTIFIED`) | +| `SQL_TXN_ISOLATION_OPTION` | `SQL_TXN_CAPABLE`, and whatever actually applies the level an application sets | + +When adding or changing a capability, look for the other info type that talks +about the same thing, and assert the relationship. +`catalog_and_schema_info_types_agree_with_each_other` and +`transaction_isolation_offers_only_the_level_sqlite_implements` are that check, +and they assert the spec's rule rather than today's values, so they keep +holding if the answer changes. + +### Transactions + +`connect` issues `PRAGMA foreign_keys = ON`. SQLite leaves it off for backward +compatibility, and the bundled library only happens to compile with +`SQLITE_DEFAULT_FOREIGN_KEYS`, so without the pragma `SQL_INTEGRITY = "Y"` +would depend on a dependency's build flags rather than on this driver. +`integrity_enhancement_facility_is_actually_enforced` checks it through +`connect`. + +SQLite supports transactions and this driver reports `SQL_TC_ALL` for +`SQL_TXN_CAPABLE`, so manual-commit mode is honoured for real: +`set_autocommit(false)` issues `BEGIN`, and `end_tran` issues `COMMIT` or +`ROLLBACK` and then opens the next transaction while still in manual-commit +mode. + +`SQL_TC_ALL` is the measured answer, not the optimistic one. The four +non-`NONE` values differ only in what DDL does inside a transaction, and the +spec separates them by observable effect: `SQL_TC_DML` means DDL "cause[s] an +error", `SQL_TC_DDL_COMMIT` that it commits, `SQL_TC_DDL_IGNORE` that it is +ignored. SQLite's DDL is transactional, so a `CREATE TABLE` between two +inserts raises nothing and a later `ROLLBACK` undoes the table along with the +rows. `transaction_capability_is_live_probed` runs exactly that and rules out +all three alternatives at once. Note that `SQL_TC_DML` is not a cautious +weaker claim: it asserts that DDL errors, so reporting it would make an +application either refuse DDL inside a transaction or commit before sending +it, silently dropping the atomicity the user asked for. + +Both `cursor_commit_behavior` and `cursor_rollback_behavior` return +`CursorBehavior::Preserve`, and **this depends on an implementation detail**: +`execute::exec_direct` materialises every result set eagerly, so no +`rusqlite::Statement` is live when `end_tran` runs. Raw SQLite is stricter: a +ROLLBACK aborts pending statements with `SQLITE_ABORT` (>= 3.7.11), which would +be `SQL_CB_CLOSE`, and a COMMIT with pending writes fails with `SQLITE_BUSY`. + +If result sets ever become lazily streamed, both hooks must be revisited, and +`SQL_CB_CLOSE` would additionally require a real +`StatementBackend::close_cursor`. That method is fallible (`Result<(), +Self::Error>`) because under `SQL_CB_CLOSE` it is the only thing that closes +the cursor during `SQLEndTran`, and a failure has to reach the statement's +diagnostic queue rather than be swallowed. Here it only resets an index into an +already-materialised `Vec`, so it cannot fail. +`end_tran_cursor_behaviour_is_preserve_for_commit_and_rollback` pins the +reported values through the FFI entry point. + +`SQL_ATTR_TXN_ISOLATION` is validated by core against `txn_isolation_options`, +which this driver answers with `SQL_TXN_SERIALIZABLE` alone. Setting any other +level on an open connection is refused with `HY024` rather than stored and +echoed back. See `txn_isolation_accepts_only_the_level_sqlite_implements`. +Because `txn_isolation_options` is a per-connection hook, a level set *before* +connecting is only checked for naming exactly one level; the comparison against +the hook happens at connect time, so an unsupported level fails the connect. + +### Cancellation + +`SQLCancel` is real: `cancel` calls `sqlite3_interrupt`, which stops the +in-flight `sqlite3_step` on that connection. + +`Backend::CancelToken` is `SqliteCancelToken`, and its two fields are scoped +differently on purpose. The interrupt handle is the *connection's*, cloned from +`connect`, because `sqlite3_interrupt` has nothing finer to aim at. The +`cancelled` flag is the *token's own*, minted fresh by `cancel_token`. Core +mints a token per statement-producing call, so a flag shared across them would +leave a cancelled statement permanently unusable, with every later error on the +connection reported as `HY008`, where the spec says "After the statement has +been canceled, the application can call SQLExecute or SQLExecDirect again." + +This is the **aliasing** token shape of the two `Backend::CancelToken`'s doc +comment describes (the token refers to the same connection the statement is +executing on), and it is sound only because SQLite documents +`sqlite3_interrupt` as safe to call from another thread. The `Arc` is core's +requirement for that shape: core clones the token out of its registry before +touching anything else, so the token has to survive a concurrent +`SQLDisconnect`. `rusqlite` already satisfies the underlying rule: its +`InterruptHandle` holds an `Arc>` shared with the +connection, and `InnerConnection::close` nulls that pointer while holding the +same mutex, so a racing `interrupt()` either finds a live handle or finds null +and does nothing. + +Three things this depends on, in order: + +- **The handle is captured in `connect`,** not fetched on demand. + `cancel_token` can neither block nor fail, and the `rusqlite::Connection` + lives behind a `Mutex`, so reaching through it would mean waiting on whatever + thread is executing. Core's own doc asks for the same thing for a different + reason: assemble the token with the connection in hand, never lazily inside + `cancel`. +- **`cancel` takes no lock this driver owns.** On `SQLCancel`'s idle path core + holds the connection's group lock across the call, so anything that waited on + it would deadlock. `interrupt()` takes only `rusqlite`'s own short-lived + interrupt lock, which no ODBC entry point holds. +- **`SQLITE_INTERRUPT` maps to `HY008`.** `map_sqlite_error` classifies + `ErrorCode::OperationInterrupted` as `SqliteError::OperationCanceled`, which + is the SQLSTATE the spec's diagnostics tables list for a statement stopped by + `SQLCancel`. Without that arm a cancelled statement would report `HY000`. + +`sql_cancel_from_another_thread_stops_a_running_statement` drives the real +entry points across two threads. It was verified by mutation: with +`token.interrupt()` removed the query runs to completion and the test fails on +the return code. Note the gate it holds: `SQLCancel`'s idle branch clears the +statement's diagnostic queue, so a cancel landing after `SQLExecDirectW` +returns would wipe the `HY008` the test is reading. + +`is_cancelled` is the other half: `cancel` signals the token's flag, this +reads it, and core turns a `true` into `HY008`. Core asks only after a backend +call has already failed, so a statement that finishes before the interrupt +lands stays successful, which the spec explicitly permits. + +### Query timeout + +`SQL_ATTR_QUERY_TIMEOUT` is enforced. `set_query_timeout` answers +`QueryTimeout::CoreCancels`, so core arms its own timer (`query_timer.rs`) and +calls `Backend::cancel` when the deadline passes. SQLite has no server-side +statement deadline, so `QueryTimeout::DataSource` is unavailable; +`CoreCancels` asserts that `cancel` really cancels, which holds here. + +The deadline covers execution rather than fetching, which is where the time +goes: `exec_direct` materialises every row before returning, so a slow `SELECT` +is slow inside that call and `SQLFetch` afterwards only walks a `Vec`. + +**`HYT00` does not come from `is_cancelled`.** Core marks its own `CancelState` +timed out before cancelling, and `QueryTimer::relabel` rewrites the failed +call's SQLSTATE ahead of the `HY008` reclassification, so the more specific +timeout wins over the cancel whatever the backend reports. +`query_timeout_stops_a_long_running_statement` was verified by mutation in both +directions: stubbing `is_cancelled` to `false` leaves it passing, while +reverting `set_query_timeout` to the default fails it on the return code. + +Core's scope caveat on `set_query_timeout` does not bite here. It warns that +the hook receives only the connection, so a backend applying the value +session-wide gives every statement the most recent one. This driver applies it +nowhere: `seconds` is ignored and core owns both the timer and the stored +value, so two statements on one connection keep their own deadlines. + +### Computed columns are typed from their values + +`sqlite3_column_decltype` names the column of a stored table or nothing at all, +so every computed column arrives with no declared type: a literal, an +expression, an aggregate, even an explicit `CAST(x AS INTEGER)`. Falling back to +`TEXT` describes `count(*)` as `SQL_WVARCHAR`, and a tool choosing a column to +sum or chart passes over it. + +`execute::infer_decl_type` supplies the missing declaration from the storage +classes of the materialised values, and `describe_column` uses it only when +SQLite offers none. Two consequences worth keeping straight: + +- **A declared type always wins.** SQLite lets any value into any column, so an + `INTEGER` column can hold text. The declaration is what the schema promises + and what the next row might hold, so inference must not reach a column that + has one. `a_declared_type_is_not_overridden_by_the_values` pins that. +- **Rows are collected before the descriptors are built.** Values are converted + using the descriptor's SQL type, so refining the type afterwards would convert + against the old one. Nothing is read twice; the rows are materialised anyway. + +The inference returns a declared-type *string* rather than a `SqlDataType`, so +precision, scale and `SQL_DESC_TYPE_NAME` all come from the same functions that +handle a real declaration and a column inferred as `INTEGER` is +indistinguishable from one declared that way. Mixed storage classes resolve to +whatever holds every value present: integers and reals to `REAL`, anything with +text to `TEXT`. NULLs are skipped, because a NULL is the absence of a value +rather than evidence of a type, and counting one would make the description +depend on which rows matched. + +### `row_count` has three answers, not two + +`StatementBackend::row_count` returns `Option`, and core reads all three +possibilities differently: + +| Answer | Means | Here | +|--------|-------|------| +| `Some(n)` | the backend counted | a searched INSERT / UPDATE / DELETE, or a materialised result set | +| `Some(-1)` | `SQL_NO_TOTAL`, cannot determine | a count exceeding `i64`; unreachable in practice | +| `None` | not applicable to this statement | DDL, transaction control, `PRAGMA`, an unexecuted prepared statement | + +The distinction between the last two is not cosmetic. Core turns a statement +with **zero columns** reporting **`Some(0)`** into `SQL_NO_DATA`, which is +`SQLExecDirect`'s documented behaviour for "a searched update, insert, or +delete statement that doesn't affect any rows". Answering `Some(0)` for DDL +would therefore make every `CREATE TABLE` return `SQL_NO_DATA`. + +SQLite offers no predicate for "is this DML" (`sqlite3_stmt_readonly` is false +for DDL too), so `execute::is_searched_dml` decides it from the statement's +leading keyword, past whitespace and both comment forms. `REPLACE` and `WITH` +count alongside the obvious three: the first is an `INSERT OR REPLACE` alias, +and the second fronts a CTE, which is only ever consulted for a zero-column +statement, so a `WITH` that declared no columns cannot be a `WITH ... SELECT`. +Being wrong is not symmetric, so an unrecognised keyword answers "no count": +withholding a count leaves `SQLRowCount` at -1, while inventing one fabricates +`SQL_NO_DATA`. + +Do **not** replace this with the number `rusqlite`'s `execute()` returns. +`sqlite3_changes()` reports the rows touched by the *most recently completed* +INSERT, UPDATE or DELETE, so a `CREATE TABLE` run after a three-row `INSERT` is +handed that `3`. `ddl_after_dml_does_not_inherit_the_dml_row_count` pins it. + +### Catalog functions + +The six catalog methods take a **typed query object** (`&TablesQuery`, +`&ColumnsQuery`, `&PrimaryKeysQuery`, `&ForeignKeysQuery`, `&StatisticsQuery`, +`&SpecialColumnsQuery`) and return **typed row vectors** (`Vec`, +`Vec`, `Vec`, `Vec`, +`Vec`, `Vec`), not a `Self::Statement`. Core +converts each row to the spec's column layout, sorts the set into the order +that function's spec page mandates, and serves it. + +Both sides are core's types and both are sealed, which is what a change in +`metadata.rs` has to work with: + +- **Neither has a struct expression here.** Every row type is + `#[non_exhaustive]`, so a row is built from `Default` and the consuming + setter per column: `TableRow::default().name(n).table_type(t)`. Each setter + takes `impl Into`, so an `Option` column accepts a bare `String`. + A column a driver does not populate is simply not named, which is the point, + since it makes a column added to a spec result set a core-only change instead + of a break in every driver. The query types are sealed the same way, with + crate-private fields, an accessor and a `with_*` setter per field, and a + `new()` for the arguments that have no honest default (`StatisticsQuery`'s + `unique_only`, `SpecialColumnsQuery`'s `identifier_type`/`scope`/`nullable`). +- **Read the filters off the query, do not destructure it.** A run of + same-typed `Option<&str>` arguments is exactly what the query types exist to + remove: `SQLForeignKeys` takes six filters, where swapping a primary-key + argument for its foreign-key counterpart would compile without complaint. + Unpacking a query back into positional arguments at the trait boundary + reintroduces that hazard one layer down, so the query travels all the way + into `metadata.rs`. +- **`TablesQuery::table_types()` is already parsed.** Core splits `TableType` + on commas and strips the optional single quotes (it is a value list, not a + pattern, and `SQL_ATTR_METADATA_ID` never applies to it), so a backend gets a + `&[String]` and never parses it. Empty means no filter. A lone `"%"` does + still arrive, because the `SQL_ALL_TABLE_TYPES` enumeration core answers + itself additionally requires the other three arguments to be empty strings; + `metadata::tables` reads that as no filter. + +Three further consequences for anything changed in `metadata.rs`: + +- **Do not sort, and do not add an `ORDER BY` for ODBC's sake.** Core sorts, + stably, on the spec's keys. A second ordering in the backend is one more + place for it to be wrong, and it overrides nothing, because core re-sorts + regardless. The one thing to keep in mind is that the sort takes NULL + placement from `Backend::null_collation`, which is why `SQLStatistics`' + table-stat row (NULL `NON_UNIQUE`) still comes first: this driver reports + `SQL_NC_LOW`. +- **Do not handle the `SQL_ALL_*` enumerations.** `SQL_ALL_CATALOGS`, + `SQL_ALL_SCHEMAS` and `SQL_ALL_TABLE_TYPES` are all the same `"%"` sentinel, + distinguished by which argument carries it while the others are empty + strings. Core detects them on the *raw* arguments before calling `tables`, + and answers from `supports_catalogs`, `supports_schemas` and `table_types`. + `catalogs` and `schemas` are left defaulted here because the first two hooks + say SQLite has neither, so core never asks. +- **A non-`Option` field is a column the spec marks "not NULL".** The types + enforce it, which is what keeps `SQLForeignKeys`' `PKCOLUMN_NAME` populated + for a `REFERENCES parent` with no column list. SQLite defines that as the + parent's primary key, so `parent_pk_column` resolves the name rather than + dropping it. + +Because ordering is core's, an ordering assertion belongs in +`ffi_integration_tests.rs`, where core's sort has actually run. The unit tests +in `metadata.rs` assert only which rows exist and what each field holds. See +`sql_statistics_w_orders_table_stat_row_first_then_unique_before_non_unique`. + +### The Windows setup dialog + +`Backend::configure_dsn` (`src/backend/setup.rs`) supplies one thing: the +dialog. Core owns the rest of `ConfigDSN`, meaning validation of *fRequest*, +rejecting `DRIVER=`, merging the data source's stored keywords in, calling +`SQLValidDSN` and writing through `SQLWriteDSNToIni`. + +The dialog itself is `packaging/windows/configure-dsn.ps1`, which also runs +standalone for a scripted install (`-NoGui -Set @{...}`). `install.bat` +installs it beside the DLL and refuses to register the driver without it. + +Nothing compiles the PowerShell, so `dsn_keys_match_the_connection_string_parser` +in `src/lib.rs` reads both the parser's `PARAM_*` constants and the dialog's +`$Fields` table out of the shipping files and asserts they offer the same keys +in both directions. A keyword added to one side and not the other fails the +build rather than surfacing as a box a user can fill in that the driver ignores. + +## Connection string keys + +Keys are matched case-insensitively and stored lowercase by core's +`ConnectParams`. + +| Key | Required | Description | +|-----|----------|-------------| +| `Database` | Yes | Path to the database file, or `:memory:` | + +Adding a key means four edits, and the build fails until the last two agree: + +1. A `PARAM_*` constant in `src/backend/types/connect_params.rs`, read in the + `TryFrom` impl. +2. The table in [`README.md`](README.md), and the one in + [`packaging/README.md`](packaging/README.md) that ships in the archive. +3. The `$Fields` table in `packaging/windows/configure-dsn.ps1`. +4. `Backend::browse_connect_attrs`, if `SQLBrowseConnect` should prompt for it. + +## Testing + +### Unit and FFI tests + +```bash +cargo test +``` + +Needs no database file: the FFI tests connect to `:memory:`. `cargo test` runs +both the per-module unit tests and `src/ffi_integration_tests.rs`, which drives +the real exported entry points against real handles. Prefer adding to the FFI +tests when the behaviour is observable by an application: they catch the +marshalling and cursor-state bugs that unit tests on the backend cannot. + +Core's `conformance` module and its connection attach/detach helpers sit behind +its default-off `test-support` feature, enabled here under `[dev-dependencies]` +so `cargo test` sees it and `cargo build` does not. It is test code that would +otherwise ship inside the driver binary. + +**Set up test data through the FFI, not by reaching into the handle.** Core's +`handles` module is `pub(crate)`, so `ConnectionHandle` and the +`rusqlite::Connection` inside it are not reachable from here. Use the +`setup_sql`, `query_scalar_i64` and `query_row_two_strings` helpers, which go +through `SQLExecDirect`/`SQLFetch`/`SQLGetData`. Each allocates its own +statement handle rather than borrowing the caller's, because the statement a +test is asserting on usually holds live state (a cursor, a prepared statement, +bound parameters) that setup would destroy. Driving setup through the driver +also means a setup path that breaks fails loudly, instead of leaving the test +asserting against an empty table. `test_support::attach_connection` is the +supported route for the different job of exercising core's connected paths +with no data source open. + +### Integration tests + +```bash +./integration-tests/setup.sh # build, create the database, write the ODBC config +./integration-tests/run-tests.sh # pyodbc suite through real unixODBC, then cargo test +./integration-tests/run-tests.sh --windows # also run the Windows VM suite +./integration-tests/run-tests.sh --skip-cargo-test # pyodbc only; what CI passes +``` + +Both are wrappers; the logic is in `integration-tests/scripts/`, with the paths +and helpers they share in `scripts/lib.sh`. Everything `setup.sh` writes lands +in `integration-tests/generated/`, which is gitignored wholesale because the +ODBC config there names absolute paths. See +[integration-tests/README.md](integration-tests/README.md) for the layout and +why the pyodbc suite is run twice. + +### Windows VM tests + +See [integration-tests/windows/WINDOWS.md](integration-tests/windows/WINDOWS.md). +Requires a provisioned libvirt VM; `integration-tests/windows/windows_test.py` +runs the same pyodbc suite over WinRM, DSN-less and then via DSN. + +### Benchmarks + +```bash +cargo bench +``` + +`benches/fetch_sqlite.rs` drives the full FFI fetch path against `:memory:` and +exists to catch regressions in the eager-materialise and per-call clone costs of +the `SqliteBackend` → `ColumnValue` → `write_column_value` pipeline. + +### What runs in core, not here + +- **Miri.** The driver crates link C libraries (bundled SQLite) that Miri cannot + execute. Core is pure Rust and holds the raw-pointer marshalling. +- **Fuzzing.** The `utf16` and `column_value` fuzz targets fuzz core's code. +- **Generic FFI entry-point tests.** Handle tags, panic safety and diagnostics + are core's. + +## Packaging and release + +`packaging/build-archives.sh` assembles the Linux and Windows release archives +from binaries already built by `cargo build --release`; see +[packaging/README.md](packaging/README.md). + +Anything destined for an archive is built with `cargo auditable`, which embeds +the dependency list `packaging/sbom.sh` generates the CycloneDX and SPDX +documents from. `sbom.sh` refuses an artifact without it, and +`packaging/test-sbom.sh` is that pipeline's own test suite. The bundled SQLite +version is declared in `packaging/sbom-native.json` and pinned against the +linked library by `the_declared_sqlite_version_is_the_one_linked` in +`src/lib.rs`. + +### Cutting a release + +`release.toml` configures `cargo-release`. It bumps the version, rewrites +`CHANGELOG.md` and `packaging/README.md`, commits, tags and pushes; the `v*` tag +triggers `.github/workflows/release.yaml`, which builds both binaries and +publishes the GitHub Release. + +```bash +release/release.sh patch # dry run +release/release.sh patch --execute # for real, from main only +``` + +`publish = false`: this crate is not published to crates.io. Tags and commits +are signed by configuration, so a release cut without a signing key fails +loudly rather than producing an unsigned tag. diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..3d90992 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,82 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +First release, so this section describes what the driver offers rather than +what changed. + +### Added + +**Querying.** An ODBC 3.80 driver for [SQLite](https://sqlite.org) on Linux and +Windows. Queries, result sets fetched a row at a time, bound parameters, and +the ODBC escape sequences `{fn ...}`, `{d ...}` and `{oj ...}` translated into +SQLite SQL. SQLite is compiled into the driver, so there is no separate library +to install and no second copy on the machine that could disagree with it. + +**Types.** SQLite is dynamically typed and has no date, time or boolean type at +all. The driver reads each column's declared type alongside the storage class +of the values in it, and maps the pair onto a proper ODBC type. That covers the +three ways SQLite users store a timestamp: ISO 8601 text, Unix epoch seconds, +and Julian day numbers. + +**Metadata.** Tables, views, columns, primary keys, foreign keys, indexes and +row identifiers, read from SQLite's own `PRAGMA` introspection and +`sqlite_master`. A tool can browse the database instead of asking you to type +table names. + +**Transactions.** Turn autocommit off and the driver opens a transaction, then +commits or rolls back on request and opens the next one. Open result sets +survive both, because every row has already been read into memory by the time +you commit. + +**Foreign keys.** SQLite enforces foreign keys only when asked, which surprises +most people who assume a `REFERENCES` clause is a rule the database keeps. The +driver issues `PRAGMA foreign_keys = ON` for every connection. + +**Cancellation and timeouts.** `SQLCancel` from another thread calls +`sqlite3_interrupt` on the connection, so a runaway query stops instead of +running to completion while the application believes it was cancelled. The +statement reports `HY008` and can be run again. `SQL_ATTR_QUERY_TIMEOUT` is +enforced the same way, reporting `HYT00` when the deadline passes, and it +covers execution, which is where a SQLite query spends its time. + +**Reported capabilities.** What a driver says about itself is how applications +decide which SQL to send, so the values here are measured rather than +transcribed. The `ALTER TABLE` clauses are established by executing each one +against the linked library, and the reserved-word list is read out of it at +runtime through `sqlite3_keyword_name`. + +**Packaging.** Installers for Linux and Windows, and a Windows dialog for +creating a data source, reachable from the ODBC Data Source Administrator's +**Add…** button. Every release artifact ships with a CycloneDX SBOM, is +published alongside an SPDX document, and is covered by `sha256sums.txt`. The +SBOM is generated from the binary's own embedded dependency list, so it +describes what was linked, including the bundled SQLite. + +### Known limitations + +- SQLite has no catalogs and no schemas, so the driver reports none rather than + inventing a one-level hierarchy. +- SQLite has no stored procedures, so those lookups return no rows. +- `SQL_C_NUMERIC` cannot be used to *retrieve* a value. A `DECIMAL` column is + described as `SQL_DECIMAL` and reads correctly as `SQL_C_CHAR` or + `SQL_C_DOUBLE`, but `SQLGetData` and a column bound to `SQL_C_NUMERIC` both + report `07006`. It works as a parameter type, so the restriction is on the + retrieval side only. +- Rows are fetched one at a time. `SQL_ATTR_ROW_ARRAY_SIZE` and + `SQL_ATTR_PARAMSET_SIZE` are both pinned at 1, so there are no block cursors + and no parameter arrays. +- Result sets are read into memory in full, which is what lets cursors survive + a commit or rollback. A `SELECT` larger than available memory will not work. +- Only the serializable isolation level is offered, because it is the only one + SQLite provides. Asking for a weaker one is refused rather than silently + ignored. +- Linux has no setup dialog. unixODBC has no convention for a driver to display + one, so a data source there is a section in `odbc.ini`. + +[Unreleased]: https://github.com/stackabletech/stackable-odbc-sqlite/commits/HEAD diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..cd1414d --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,66 @@ +# Project Rules + +Read and follow @AGENTS.md. It holds the architecture, the patterns and the +procedures, and it is where the reasoning behind every rule below lives. + +## Non-Negotiable Rules + +- **ODBC spec compliance is mandatory.** Read the spec page for every function + whose behaviour you change. The generic FFI entry points live in + `stackable-odbc-core`, but what this driver returns from `get_info`, + `get_info_raw`, the catalog functions and the type-conversion paths is + directly observable by applications, and each has a spec-defined shape and + value range. Never claim a SQLSTATE or an info value is wrong without + checking the actual spec table first. Pay attention to **(DM)** annotations: + those SQLSTATEs are returned by the Driver Manager, not the driver. +- **Route every client error through `map_sqlite_error`.** It is the single + place that decides the SQLSTATE, and a new classified variant must carry the + originating error in its `cause` field. See + [Backend error mapping](AGENTS.md#backend-error-mapping). +- **One error type.** Every `Backend` and `StatementBackend` method returns + `Result<_, SqliteError>`. An `OdbcError` core produced travels back through + `SqliteError::Odbc` via `.into()`. Never reclassify it, which would discard + the SQLSTATE core chose. +- **Declare each capability once.** A `SQLGetInfo` value with a `Backend` hook + is answered through the hook only, never also in `get_info_raw`. See + [Declaring capabilities](AGENTS.md#declaring-capabilities). +- **Use `odbc-sys` types**, re-exported from `stackable_odbc_core::types`, or + from `stackable_odbc_core::odbc_sys` for anything `types` does not re-export. + Never redefine what it provides, never add an `odbc-sys` dependency to this + crate's `Cargo.toml`, and never hand-roll a `#[repr(C)]` mirror of one of its + structs. See [Named constants](AGENTS.md#named-constants). +- **Convert raw integers to typed enums at the boundary** with core's + `xxx_from_raw()` functions, never `transmute`. +- **Do not make result-set fetching lazy.** `exec_direct` materialises every + row before returning, and two reported ODBC capabilities + (`SQL_CURSOR_COMMIT_BEHAVIOR`, `SQL_CURSOR_ROLLBACK_BEHAVIOR`) are only + correct because of it. See + [Result sets are materialised eagerly](AGENTS.md#result-sets-are-materialised-eagerly) + and [Transactions](AGENTS.md#transactions). +- **Run `pre-commit run --all-files` before every commit.** It is the single + source of truth for what must pass. + +## Scope + +- Do not modify files outside the scope of the current task. +- Do not add features, refactoring, or "improvements" beyond what was asked. +- If unsure whether something is in scope, ask. + +## Data Retrieval + +Never read entire files by default. Survey, locate, then extract. + +1. Survey first. Check the file size with `stat -c%s file` before reading it. + Anything over 50 KB must be sliced, not read whole; several modules in + `src/` are. +2. Navigate definitions with ctags. Run `ctags -R .` once to build the index, + then `grep "^SymbolName" tags` for the exact file and line of any function, + struct or trait. No file reading needed. +3. Locate with Grep. Find patterns, keywords or usages before reading. Use `-C` + for context lines. +4. Extract with Read, using `offset` and `limit` once you know the line range. +5. Read structured data with a tool that understands it: `jq` for JSON, `yq` + for YAML. Never read raw markup whole. +6. Survey the filesystem with `tree -L 2 -I '.git|target|node_modules'`, not a + recursive `ls`. +7. Verify edits with `git diff -u` rather than re-reading the file. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..a09807d --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,153 @@ +# Contributing + +Thanks for considering a contribution. Bug reports, connection strings that +fail, and reports of a tool that will not talk to the driver are all useful. + +This driver exists to exercise +[`stackable-odbc-core`](https://github.com/stackabletech/stackable-odbc-core) on +a backend that needs no server, so a change that makes it a better test of core +is as welcome as one that makes it a better SQLite driver. + +- **Questions and ideas:** [GitHub Discussions](https://github.com/orgs/stackabletech/discussions) + or [Discord](https://discord.gg/7kZ3BNnCAF). +- **Bugs:** open an issue. Please say which platform, which Driver Manager + (unixODBC or the Windows one), and which application. A driver log helps most + of all: set `ODBC_LOG_FILE` and `ODBC_LOG_LEVEL=debug` and attach the result. +- **Security problems:** do not open an issue. See [SECURITY.md](SECURITY.md). + +## Building + +You need the unixODBC development libraries, because the ODBC bindings link +against them. SQLite itself is compiled into the driver, so there is nothing +else to install, and no database or ODBC configuration is needed to build and +run the unit tests. + +```bash +sudo apt-get install unixodbc-dev # Debian/Ubuntu +sudo pacman -S unixodbc # Arch +``` + +```bash +git clone https://github.com/stackabletech/stackable-odbc-sqlite +cd stackable-odbc-sqlite +cargo build --release +``` + +That produces `target/release/libstackable_odbc_sqlite.so`. + +Everything generic about being an ODBC driver lives in +[`stackable-odbc-core`](https://github.com/stackabletech/stackable-odbc-core): +handle management, UTF-16 marshalling, diagnostics, panic safety and the +exported C entry points. This repository holds only the SQLite-specific half. +Cargo fetches core for you, so there is nothing to clone by hand. + +### Working on core at the same time + +This is the common case here, because this driver is where core's changes are +tried out. To build against a local checkout of core rather than the fetched +one, add a `[patch]` to your own `.cargo/config.toml`, which is not checked in: + +```toml +[patch."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/stackabletech/stackable-odbc-core.git"] +stackable-odbc-core = { path = "../stackable-odbc-core" } +``` + +`.cargo/` is gitignored, so the override cannot be committed. **`Cargo.lock` +can**: cargo rewrites core's entry to the local path while the patch is active, +so check `git status` before committing. Remove the override, or push your core +changes, before you rely on a build. + +The toolchain version is pinned in `rust-toolchain.toml`, so rustup will fetch +the right one on first build. + +### Windows + +Cross-compile with MinGW (`gcc-mingw-w64-x86-64`): + +```bash +rustup target add x86_64-pc-windows-gnu +cargo build --release --target x86_64-pc-windows-gnu +``` + +That produces `target/x86_64-pc-windows-gnu/release/stackable_odbc_sqlite.dll`. +`build.rs` embeds a version resource into it with `windres`, which comes with +that same package; a build without it fails rather than shipping a DLL the ODBC +Data Source Administrator lists as `Not marked`. + +Anything destined for a release archive is built with +[`cargo auditable`](https://github.com/rust-secure-code/cargo-auditable), which +embeds the dependency list the SBOM is generated from. See +[`packaging/README.md`](packaging/README.md). + +## Testing + +```bash +cargo test # unit and FFI tests; no setup needed +cargo clippy --all-targets -- -D warnings +``` + +`cargo test` must produce zero warnings. It drives the real exported C entry +points against real handles, so it catches marshalling bugs an ordinary Rust +test cannot. + +The integration suite goes one layer further out, through real unixODBC using +Python's `pyodbc`. It needs no server, so it runs on every pull request: + +```bash +./integration-tests/setup.sh # build the driver, create the database, write the ODBC config +./integration-tests/run-tests.sh # run the pyodbc suite, then cargo test +``` + +See [`integration-tests/README.md`](integration-tests/README.md) for the flags. +The Windows suite runs the same tests through the Windows Driver Manager in a +VM; see +[`integration-tests/windows/WINDOWS.md`](integration-tests/windows/WINDOWS.md). + +## Before you commit + +```bash +pre-commit run --all-files +``` + +That is the gate, and it is the single source of truth for what must pass. It +runs rustfmt, clippy, `cargo test`, rustdoc, cargo-deny, cargo-sort, shellcheck +and markdownlint. + +Two more things a change usually needs: + +- **A changelog entry**, under `## [Unreleased]` in + [`CHANGELOG.md`](CHANGELOG.md), if an ODBC application can observe the + difference. A changed SQLSTATE, a changed `SQLGetInfo` value, a new + connection-string key or a different type mapping all count. +- **A new connection-string key touches four places**: the parser in + `src/backend/types/connect_params.rs`, the key tables in + [`README.md`](README.md) and [`packaging/README.md`](packaging/README.md), and + the `$Fields` table in `packaging/windows/configure-dsn.ps1`. + `dsn_keys_match_the_connection_string_parser` in `src/lib.rs` fails the build + if the parser and the dialog disagree. See + [Connection string keys](AGENTS.md#connection-string-keys). + +## Where things live + +[`AGENTS.md`](AGENTS.md) is the working reference: module layout, the split +against `stackable-odbc-core`, the error-mapping rules, and the measured SQLite +and Driver Manager behaviour behind the design decisions. Read the section that +covers whatever you are about to change. It is written for AI coding agents and +human contributors alike. + +Two rules are worth stating here, because they are the ones most easily broken +by a reasonable-looking change: + +- **Read the ODBC spec page for any function whose behaviour you change.** What + the driver returns from `SQLGetInfo`, from the catalog functions and from the + type-conversion paths is directly observable by applications, and each has a + spec-defined shape and value range. +- **Route every client error through `map_sqlite_error`.** It is the single + place that decides the SQLSTATE and carries SQLite's own extended result code + through to `SQLGetDiagRec`. Building an error at the call site quietly + degrades it. + +## License + +By contributing you agree that your contribution is licensed under +[Apache-2.0](LICENSE). diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..74b79f5 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,1251 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "alloca" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5a7d05ea6aea7e9e64d25b9156ba2fee3fdd659e34e41063cd2fc7cd020d7f4" +dependencies = [ + "cc", +] + +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "bit-set" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + +[[package]] +name = "cc" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "clap" +version = "4.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "301b56658598e48f3648647ac6fc887be7e7108eddfa4e9b63fcf3ec58c0cadf" +dependencies = [ + "clap_builder", +] + +[[package]] +name = "clap_builder" +version = "4.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94a65403d1a1bd28f7dc68eb8506e8874808ee5eecb59298de588e2e1407a078" +dependencies = [ + "anstyle", + "clap_lex", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "criterion" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "950046b2aa2492f9a536f5f4f9a3de7b9e2476e575e05bd6c333371add4d98f3" +dependencies = [ + "alloca", + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "itertools", + "num-traits", + "oorandom", + "page_size", + "plotters", + "rayon", + "regex", + "serde", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8d80a2f4f5b554395e47b5d8305bc3d27813bacb73493eb1001e8f76dae29ea" +dependencies = [ + "cast", + "itertools", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d85363c37faeca707aef026efa9f3b34d077bce547e48f770770625c6013679e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" + +[[package]] +name = "either" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "futures-core" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" + +[[package]] +name = "futures-task" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109" + +[[package]] +name = "futures-util" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi 5.3.0", + "wasip2", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hashlink" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32069d97bb81e38fa67eab65e3393bf804bb85969f2bc06bf13f64aef5aba248" +dependencies = [ + "hashbrown 0.17.1", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "libsqlite3-sys" +version = "0.38.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6c19a05435c21ac299d71b6a9c13db3e3f47c520517d58990a462a1397a61db" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "log" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "odbc-sys" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "245cb4fe8236df4fd352ba96075d754233c6509d654d9f1c1482158b7d6c083d" + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + +[[package]] +name = "page_size" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "proptest" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b45fcc2344c680f5025fe57779faef368840d0bd1f42f216291f0dc4ace4744" +dependencies = [ + "bit-set", + "bit-vec", + "bitflags", + "num-traits", + "rand", + "rand_chacha", + "rand_xorshift", + "regex-syntax", + "rusty-fork", + "tempfile", + "unarray", +] + +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +dependencies = [ + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "rand_xorshift" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "513962919efc330f829edb2535844d1b912b0fbe2ca165d613e4e8788bb05a5a" +dependencies = [ + "rand_core", +] + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rsqlite-vfs" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c51c9ae4df8a7fba42103df5c621fa3c37eccf3a3c650879e90fc48b11cc192c" +dependencies = [ + "hashbrown 0.16.1", + "thiserror", +] + +[[package]] +name = "rusqlite" +version = "0.40.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11438310b19e3109b6446c33d1ed5e889428cf2e278407bc7896bc4aaea43323" +dependencies = [ + "bitflags", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "smallvec", + "sqlite-wasm-rs", +] + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "rusty-fork" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc6bf79ff24e648f6da1f8d1f011e9cac26491b619e6b9280f2b47f1774e6ee2" +dependencies = [ + "fnv", + "quick-error", + "tempfile", + "wait-timeout", +] + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "snafu" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e45cb604038abb7b926b679887b3226d8d0f23874b66623625a0454be425a4b7" +dependencies = [ + "snafu-derive", +] + +[[package]] +name = "snafu-derive" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "287f59010008f0d7cf5e3b03196d666c1acc46c8d3e9cf34c28a1a7157601e72" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "sqlite-wasm-rs" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc3efc0da82635d7e1ced0053bbbfa8c7ab9645d0bf36ceb4f7127bb85315d75" +dependencies = [ + "cc", + "js-sys", + "rsqlite-vfs", + "wasm-bindgen", +] + +[[package]] +name = "stackable-odbc-core" +version = "0.1.0" +source = "git+https://github.com/stackabletech/stackable-odbc-core.git?tag=v0.1.0#23c924489e135d1d3da1d1664ae16bf8656d5aa3" +dependencies = [ + "odbc-sys", + "snafu", + "tracing", + "tracing-appender", + "tracing-subscriber", +] + +[[package]] +name = "stackable-odbc-sqlite" +version = "0.0.1" +dependencies = [ + "criterion", + "proptest", + "rusqlite", + "serde_json", + "snafu", + "stackable-odbc-core", + "tracing", +] + +[[package]] +name = "symlink" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7973cce6668464ea31f176d85b13c7ab3bba2cb3b77a2ed26abd7801688010a" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "thiserror" +version = "2.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09a43598840e33d5b0331f38c5e30d13bb11c11210a4b58f0d9b18a5a5eefcd9" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "thread_local" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "time" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + +[[package]] +name = "time-macros" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-appender" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "050686193eb999b4bb3bc2acfa891a13da00f79734704c4b8b4ef1a10b368a3c" +dependencies = [ + "crossbeam-channel", + "symlink", + "thiserror", + "time", + "tracing-subscriber", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "unarray" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "wait-timeout" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ac3b126d3914f9849036f826e054cbabdc8519970b8998ddaf3b5bd3c65f11" +dependencies = [ + "libc", +] + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.119", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8622dcb61c0bcc9fffa6938bed81210af2da9a7e4a1a834b2e37a59b6dfb6141" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "zerocopy" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..e4bca41 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,53 @@ +[package] +name = "stackable-odbc-sqlite" +version = "0.0.1" +edition = "2024" +rust-version = "1.95.0" +authors = ["Stackable GmbH "] +license = "Apache-2.0" +description = "ODBC driver for SQLite, built on the stackable-odbc-core framework." +repository = "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/stackabletech/stackable-odbc-sqlite" +readme = "README.md" +keywords = ["odbc", "sqlite", "driver", "ffi", "sql"] +categories = ["database", "external-ffi-bindings", "api-bindings"] + +[lib] +crate-type = ["cdylib", "rlib"] + +[dependencies] +# "limits" is dependency-free; it exposes sqlite3_limit, which SQL_MAX_* is +# read from rather than hardcoded. +# +# "column_metadata" exposes sqlite3_table_column_metadata, which is the only +# way to answer SQLDescribeCol's nullability and SQL_DESC_BASE_TABLE_NAME +# truthfully: it reports a result column's originating table and its NOT NULL +# constraint, and reports neither for a computed column -- which is exactly the +# SQL_NULLABLE_UNKNOWN case. +rusqlite = { version = "0.40", features = [ + "bundled", + "column_decltype", + "column_metadata", + "limits", +] } +serde_json = "1" +snafu = "0.9" +stackable-odbc-core = { git = "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/stackabletech/stackable-odbc-core.git", tag = "v0.1.0" } +tracing = "0.1" + +[dev-dependencies] +criterion = { version = "0.8", features = ["html_reports"] } +proptest = "1" +# "test-support" gates core's `conformance` module and the connection +# attach/detach helpers. Default-off there because it is test code that would +# otherwise land in this driver's shipped binary; enabled only here, so +# `cargo test` sees it and `cargo build` does not. +stackable-odbc-core = { git = "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/stackabletech/stackable-odbc-core.git", tag = "v0.1.0", features = ["test-support"] } + +[lints.clippy] +unwrap_in_result = "deny" +unwrap_used = "deny" +panic = "deny" + +[[bench]] +name = "fetch_sqlite" +harness = false diff --git a/README.md b/README.md new file mode 100644 index 0000000..f752478 --- /dev/null +++ b/README.md @@ -0,0 +1,234 @@ + + +

+ Stackable Logo +

+ +

Stackable ODBC Driver for SQLite

+ +

Open a SQLite file from Excel, DBeaver, LibreOffice or Python, with no server to run.

+ +[![Build and Test](https://github.com/stackabletech/stackable-odbc-sqlite/actions/workflows/build.yaml/badge.svg)](https://github.com/stackabletech/stackable-odbc-sqlite/actions/workflows/build.yaml) +[![Security Audit](https://github.com/stackabletech/stackable-odbc-sqlite/actions/workflows/security_audit.yaml/badge.svg)](https://github.com/stackabletech/stackable-odbc-sqlite/actions/workflows/security_audit.yaml) +[![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/stackabletech/stackable-odbc-sqlite/badge)](https://scorecard.dev/viewer/?uri=github.com/stackabletech/stackable-odbc-sqlite) +[![PRs Welcome](https://img.shields.io/badge/PRs-welcome-green.svg)](https://docs.stackable.tech/home/stable/contributor/index.html) +[![Apache License 2.0](https://img.shields.io/badge/license-Apache--2.0-green)](./LICENSE) +[![ODBC 3.80](https://img.shields.io/badge/ODBC-3.80-blue)](#compatibility) +[![Platforms](https://img.shields.io/badge/platforms-Linux%20%7C%20Windows-blue)](#compatibility) +[![SQLite bundled](https://img.shields.io/badge/SQLite-3.53.2%20bundled-blue)](https://sqlite.org) + +[Stackable Data Platform](https://stackable.tech/) | [Platform Docs](https://docs.stackable.tech/) | [Discussions](https://github.com/orgs/stackabletech/discussions) | [Discord](https://discord.gg/7kZ3BNnCAF) + +## What is this? + +[SQLite](https://sqlite.org) is a database that lives in a single file. There +is nothing to install and nothing to start, because the whole database is one +`.db` file you can copy onto a USB stick. Your phone is running several of them +right now. + +Most desktop tools cannot open one of those files directly, but nearly all of +them speak **ODBC**, a standard that lets any tool load a small library, called +a driver, and talk to a database through it. + +This is the ODBC driver for SQLite. Install it, and Excel, LibreOffice Base, +DBeaver, `isql` and Python's `pyodbc` can query a SQLite file as if it were a +full database server. Linux and Windows are both first-class targets. + +SQLite itself is compiled into the driver, so there is nothing else to install +and no second copy on the machine that could disagree with it. + +## Quick start + +Download an archive from the +[releases page](https://github.com/stackabletech/stackable-odbc-sqlite/releases). + +### Windows + +1. Unzip `stackable-odbc-sqlite--windows-x64.zip`. +2. Right-click `install.bat` and choose **Run as administrator**. This registers + the driver with Windows. +3. Open **ODBC Data Sources (64-bit)** from the Start menu, click **Add**, and + pick `stackable_odbc_sqlite` from the list. Name the data source, browse to + your `.db` file, and click **Test connection** before saving. + +Step 3 creates a *DSN*: a saved connection with a name. Once it exists, every +tool on the machine can pick it from a list instead of asking you to type a +connection string. + +### Linux + +You need unixODBC (the `unixodbc` package). Installing the driver registers it +system-wide, so it needs root. + +```bash +mkdir /tmp/sqlite-odbc +tar xzf stackable-odbc-sqlite--linux-x64.tar.gz -C /tmp/sqlite-odbc +cd /tmp/sqlite-odbc +sudo ./install.sh +``` + +Check it worked with `odbcinst -q -d`, which should list +`[stackable_odbc_sqlite]`. + +### Your first query + +```python +import pyodbc + +conn = pyodbc.connect("Driver=stackable_odbc_sqlite;Database=/path/to/your.db") +for row in conn.cursor().execute("SELECT name FROM sqlite_master WHERE type = 'table'"): + print(row.name) +``` + +For the full install and uninstall reference, see +[`packaging/README.md`](packaging/README.md). + +## Connecting + +Connection strings are `Key=Value` pairs joined by `;`. Keys are +case-insensitive. There is exactly one key. + +| Key | Required | Meaning | +|-----|----------|---------| +| `Database` | Yes | Path to the SQLite file, or `:memory:` for a throwaway in-memory database | + +```text +Driver=stackable_odbc_sqlite;Database=/path/to/your.db +``` + +Instead of typing that every time you can save it as a DSN, which is a named, +stored connection much like a browser bookmark. On Linux, add a section to +`~/.odbc.ini`: + +```ini +[SQLite Test] +Driver = stackable_odbc_sqlite +Database = /path/to/your.db +``` + +On Windows the **Add** button in the ODBC Data Source Administrator writes one +for you. See [`packaging/README.md`](packaging/README.md) for that and for the +scripted alternatives. + +## What you get + +- **The stop button stops the query.** Cancelling from your tool calls SQLite's + `sqlite3_interrupt` on the connection, so a runaway query really stops rather + than running to the end while your tool reports it as cancelled. The + statement can be run again afterwards. Query timeouts work the same way, so + "give up after 30 seconds" is a promise the driver keeps. + +- **Real transactions.** Turn autocommit off and the driver opens a transaction + for you, then commits or rolls back when you say so and immediately opens the + next one. Your open result sets survive both, because the driver has already + read every row into memory by the time you commit. + +- **Foreign keys are switched on.** SQLite ships with foreign-key enforcement + *off* for backwards compatibility, which surprises almost everyone. This + driver turns it on for every connection, so a `REFERENCES` clause in your + schema is a rule the database keeps. + +- **Your tool can browse the database.** Tables, views, columns, primary keys, + foreign keys, indexes and row identifiers all show up in the object browser, + read from SQLite's own `PRAGMA` introspection, so you can click through what + is there instead of guessing table names. + +- **Columns get sensible types even though SQLite has almost none.** SQLite is + dynamically typed. Any value can go in any column, and there is no `DATE` or + `BOOLEAN` type at all. The driver reads each column's declared type together + with the storage class of its values and maps the pair onto a proper ODBC + type. That covers the three ways people store a timestamp in SQLite: ISO + text, Unix seconds and Julian day numbers. + +- **Your tool gets accurate answers about what SQLite supports.** Applications + choose which SQL to send based on what the driver reports about itself, so + those answers are measured against the bundled library rather than copied + from documentation. The `ALTER TABLE` clauses are checked by executing each + one, and the reserved-word list is read out of the library at runtime. + +- **Windows gets its own installer and setup dialog**, so the ODBC + administrator's **Add** button behaves the way it does for a commercial + driver. The DLL is cross-compiled, export-checked and unit-tested on every + pull request, and the integration suite can also be run through the Windows + Driver Manager in a VM. + +- **Every release says what is inside it.** Both archives carry a CycloneDX + SBOM generated from the binary's own embedded dependency list rather than + from `Cargo.toml`, so it describes what was actually linked, including the + bundled SQLite. The release page also carries SPDX documents, checksums and + build provenance attestations. + +## Limits + +Each of these is reported to your tool as unsupported rather than quietly +faked, so the tool can react instead of trusting a wrong answer. + +- **No catalogs and no schemas.** SQLite has neither, so the driver says so + rather than inventing a one-level hierarchy for the sake of looking familiar. +- **No stored procedures.** SQLite has none, so those lookups return nothing. +- **Rows arrive one at a time.** There are no block cursors and no parameter + arrays. +- **Result sets are read into memory in one go.** That is what lets cursors + survive a commit or rollback, but a `SELECT` over a table larger than your + RAM will not work. +- **One isolation level.** SQLite gives you serializable transactions, so that + is the only level offered, and asking for a weaker one is refused up front. +- **No setup dialog on Linux.** unixODBC has no convention for a driver to put + a window on the screen, so on Linux a DSN is defined by a section in `odbc.ini`. + +## Compatibility + +| Component | Support | +|---|---| +| ODBC | 3.80 | +| Platforms | Linux x86-64, Windows x86-64 | +| Driver Managers | unixODBC, and the Windows Driver Manager | +| SQLite | 3.53.2, compiled into the driver | +| Tested with | `pyodbc`, `isql` | + +## Troubleshooting + +**Turn on logging first.** The driver logs to a file when you ask it to, and +that is usually enough to see what a tool is really sending: + +```bash +export ODBC_LOG_LEVEL=debug # trace, debug, info, warn, error +export ODBC_LOG_FILE=/tmp/sqlite-odbc.log +``` + +On Windows, set the same two as environment variables. The log may contain your +SQL, so check it before sharing. + +**You connected fine but the database is empty.** SQLite creates a file that +does not exist yet rather than refusing, so a typo in the path connects +successfully and finds nothing. Check the path. The Windows dialog's **Test +connection** button reports the table count for exactly this reason. + +**The driver does not appear in the list.** On Linux, run `odbcinst -q -d`; if +`[stackable_odbc_sqlite]` is missing, the install did not complete. On Windows, +make sure you opened **ODBC Data Sources (64-bit)**: a 64-bit driver is +invisible to the 32-bit Administrator, and both are in the Start menu under +similar names. + +**A `REFERENCES` clause is being enforced that was not before.** That is +deliberate. The driver turns foreign-key enforcement on for every connection, +which most other SQLite tooling leaves off. + +## Getting help + +- [GitHub Discussions](https://github.com/orgs/stackabletech/discussions) for + questions +- [Discord](https://discord.gg/7kZ3BNnCAF) to talk to us +- [Issues](https://github.com/stackabletech/stackable-odbc-sqlite/issues) for + bugs, and [SECURITY.md](SECURITY.md) for anything security-related + +## Contributing + +See [CONTRIBUTING.md](CONTRIBUTING.md) for building from source, running the +tests, and how the repository is laid out. [AGENTS.md](AGENTS.md) has the +architecture and the ODBC design rationale behind what the driver reports. +[CHANGELOG.md](CHANGELOG.md) records what changed in each release. + +## License + +Apache-2.0 diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..03667f7 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,38 @@ +# Security Policy + +## Reporting a Vulnerability + +Please report security vulnerabilities privately, not through a public issue. + +The preferred channel is GitHub's private vulnerability reporting: open the +**Security** tab of this repository and choose **Report a vulnerability**. This +reaches the maintainers directly and keeps the report confidential until a fix +is available. + +If you cannot use that channel, email `info@stackable.tech` with `SECURITY` in +the subject line. + +Please include the driver version, the platform and Driver Manager in use, and +the steps needed to reproduce the issue. + +Note that SQLite itself is compiled into the driver rather than loaded from the +system, so an advisory against SQLite applies to whichever version this driver +bundles, not to the one installed on the machine. The bundled version is +recorded in every release archive's SBOM. + +## What to Expect + +We aim to acknowledge a report within three working days and to give an initial +assessment within ten. We will keep you informed while a fix is prepared, and we +will credit you in the advisory unless you ask us not to. + +## Supported Versions + +Security fixes are made against the most recent release and the `main` branch. +While the driver is below 1.0, fixes are not backported to earlier releases: +upgrade to the current release to receive them. + +## Disclosure + +Fixed vulnerabilities are published as GitHub Security Advisories against this +repository, naming the affected versions and the release that carries the fix. diff --git a/benches/fetch_sqlite.rs b/benches/fetch_sqlite.rs new file mode 100644 index 0000000..0c83fd2 --- /dev/null +++ b/benches/fetch_sqlite.rs @@ -0,0 +1,569 @@ +//! End-to-end fetch-path benchmarks for the SQLite backend. +//! +//! Goes through the full FFI path: alloc handles -> connect (`:memory:`) -> +//! exec_direct -> fetch -> {bind_col | get_data} -> cleanup. Catches the +//! eager-materialize + per-call clone cost in the SqliteBackend → ColumnValue +//! → write_column_value pipeline. +//! +//! Two workload shapes, matching core's own fetch-throughput benchmark: +//! * Shape A: mixed columns (BENCH_ROWS × BENCH_COLS, 50/40/10 i64/str/decimal) +//! * Shape B: 5 columns × BENCH_WIDE_STR_LEN-char strings (BENCH_WIDE_ROWS rows) +//! +//! Three scenarios: +//! * late_binding: SQLFetch + per-cell SQLGetData +//! * bound_columns: SQLBindCol + SQLFetch +//! * repeat_get_data: SQLGetData called BENCH_REPEAT_GET_DATA times per cell +//! +//! Run: +//! cargo bench +//! BENCH_ROWS=1000000 cargo bench + +use std::ffi::c_void; +use std::hint::black_box; +use std::time::Duration; + +use criterion::{BatchSize, BenchmarkId, Criterion, Throughput, criterion_group, criterion_main}; +use stackable_odbc_core::ffi; +use stackable_odbc_core::types::{CDataType, HandleType, SqlReturn}; +use stackable_odbc_sqlite::SqliteBackend; + +#[derive(Clone, Copy)] +struct BenchConfig { + rows: usize, + cols: usize, + wide_rows: usize, + wide_str_len: usize, + repeat_get_data: usize, +} + +fn env_or(name: &str, default: T) -> T { + std::env::var(name) + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or(default) +} + +fn bench_config() -> BenchConfig { + BenchConfig { + rows: env_or("BENCH_ROWS", 100_000), + cols: env_or("BENCH_COLS", 20), + wide_rows: env_or("BENCH_WIDE_ROWS", 10_000), + wide_str_len: env_or("BENCH_WIDE_STR_LEN", 1024), + repeat_get_data: env_or("BENCH_REPEAT_GET_DATA", 3), + } +} + +fn configure_for_size(c: Criterion, rows: usize) -> Criterion { + if rows > 250_000 { + c.sample_size(20) + .measurement_time(Duration::from_secs(30)) + .warm_up_time(Duration::from_secs(3)) + } else { + c + } +} + +fn shape_a_split(n_cols: usize) -> (usize, usize, usize) { + let s = (n_cols * 4) / 10; + let d = n_cols / 10; + let i = n_cols - s - d; + debug_assert_eq!(i + s + d, n_cols, "shape_a_split must total n_cols"); + (i, s, d) +} + +/// Allocate env + conn + stmt handles. Caller must call `cleanup` exactly once. +unsafe fn alloc_handles() -> (*mut c_void, *mut c_void, *mut c_void) { + unsafe { + let mut env: *mut c_void = std::ptr::null_mut(); + let _ = ffi::handle::sql_alloc_handle::( + HandleType::Env as i16, + std::ptr::null_mut(), + &mut env, + ); + let mut conn: *mut c_void = std::ptr::null_mut(); + let _ = + ffi::handle::sql_alloc_handle::(HandleType::Dbc as i16, env, &mut conn); + let mut stmt: *mut c_void = std::ptr::null_mut(); + let _ = ffi::handle::sql_alloc_handle::( + HandleType::Stmt as i16, + conn, + &mut stmt, + ); + (env, conn, stmt) + } +} + +unsafe fn connect_memory(conn: *mut c_void) -> SqlReturn { + unsafe { + let wide: Vec = "Database=:memory:".encode_utf16().collect(); + ffi::connect::sql_driver_connect_w::( + conn, + std::ptr::null_mut(), + wide.as_ptr(), + wide.len() as i16, + std::ptr::null_mut(), + 0, + std::ptr::null_mut(), + 0, + ) + } +} + +unsafe fn exec_direct(stmt: *mut c_void, sql: &str) -> SqlReturn { + unsafe { + let wide: Vec = sql.encode_utf16().collect(); + ffi::execute::sql_exec_direct_w::(stmt, wide.as_ptr(), wide.len() as i32) + } +} + +unsafe fn cleanup(env: *mut c_void, conn: *mut c_void, stmt: *mut c_void) { + unsafe { + let _ = ffi::handle::sql_free_handle::(HandleType::Stmt as i16, stmt); + let _ = ffi::connect::sql_disconnect::(conn); + let _ = ffi::handle::sql_free_handle::(HandleType::Dbc as i16, conn); + let _ = ffi::handle::sql_free_handle::(HandleType::Env as i16, env); + } +} + +/// Run setup SQL on `conn` through `SQLExecDirect`. +/// +/// Setup goes through ODBC dispatch rather than reaching for the underlying +/// `rusqlite::Connection`, which core's `handles` module does not expose in any +/// case. Every setup here is three statements (`DROP`, `CREATE` and one bulk +/// `INSERT` whose rows are generated by a recursive CTE inside SQLite), so the +/// dispatch is paid three times, not once per row, and setup runs outside the +/// measured section regardless. +/// +/// `SQLExecDirect` executes one statement, hence the split on `;`; none of the +/// generated setup SQL contains a `;` inside a string literal. +unsafe fn exec_setup(conn: *mut c_void, sql: &str) { + unsafe { + let mut stmt: *mut c_void = std::ptr::null_mut(); + let ret = ffi::handle::sql_alloc_handle::( + HandleType::Stmt as i16, + conn, + &mut stmt, + ); + assert_eq!(ret, SqlReturn::SUCCESS, "alloc setup stmt"); + for one in sql.split(';').map(str::trim).filter(|s| !s.is_empty()) { + let wide: Vec = one.encode_utf16().collect(); + let ret = ffi::execute::sql_exec_direct_w::( + stmt, + wide.as_ptr(), + wide.len() as i32, + ); + assert_eq!(ret, SqlReturn::SUCCESS, "setup sql: {one}"); + } + let _ = ffi::handle::sql_free_handle::(HandleType::Stmt as i16, stmt); + } +} + +/// Build a `CREATE TABLE` + bulk `INSERT` for Shape A. +fn shape_a_setup_sql(rows: usize, cols: usize) -> String { + let (n_i, n_s, n_d) = shape_a_split(cols); + let mut col_defs: Vec = Vec::with_capacity(cols); + for i in 0..n_i { + col_defs.push(format!("i{i} INTEGER")); + } + for i in 0..n_s { + col_defs.push(format!("s{i} TEXT")); + } + for i in 0..n_d { + col_defs.push(format!("d{i} TEXT")); + } + let create = format!( + "DROP TABLE IF EXISTS bench_a; CREATE TABLE bench_a ({});", + col_defs.join(", ") + ); + + // Generate the row data via SQLite's recursive CTE so we don't need 100k INSERT statements. + let select_exprs: Vec = { + let mut v = Vec::with_capacity(cols); + for i in 0..n_i { + v.push(format!("(n + {i})")); + } + for _ in 0..n_s { + v.push("'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'".into()); + } + for _ in 0..n_d { + v.push("'12345678.90'".into()); + } + v + }; + let insert = format!( + "INSERT INTO bench_a SELECT {} FROM ( + WITH RECURSIVE seq(n) AS (SELECT 1 UNION ALL SELECT n+1 FROM seq WHERE n < {}) + SELECT n FROM seq + );", + select_exprs.join(", "), + rows, + ); + format!("{create} {insert}") +} + +/// Build a `CREATE TABLE` + bulk `INSERT` for Shape B. +fn shape_b_setup_sql(rows: usize, str_len: usize) -> String { + let s = "x".repeat(str_len); + let create = "DROP TABLE IF EXISTS bench_b; CREATE TABLE bench_b (s0 TEXT, s1 TEXT, s2 TEXT, s3 TEXT, s4 TEXT);".to_string(); + let insert = format!( + "INSERT INTO bench_b SELECT '{s}', '{s}', '{s}', '{s}', '{s}' FROM ( + WITH RECURSIVE seq(n) AS (SELECT 1 UNION ALL SELECT n+1 FROM seq WHERE n < {rows}) + SELECT n FROM seq + );" + ); + format!("{create} {insert}") +} + +fn shape_a_select(cols: usize) -> String { + let (n_i, n_s, n_d) = shape_a_split(cols); + let mut names: Vec = Vec::with_capacity(cols); + for i in 0..n_i { + names.push(format!("i{i}")); + } + for i in 0..n_s { + names.push(format!("s{i}")); + } + for i in 0..n_d { + names.push(format!("d{i}")); + } + format!("SELECT {} FROM bench_a", names.join(", ")) +} + +fn shape_b_select() -> String { + "SELECT s0, s1, s2, s3, s4 FROM bench_b".to_string() +} + +/// Late-binding drain: SQLFetch + per-cell SQLGetData into a discard buffer. +unsafe fn drain_late_binding(stmt: *mut c_void, n_cols: u16) -> usize { + let mut buf = vec![0u8; 4096]; // generous per-cell scratch buffer + let mut ind: isize = 0; + let mut count = 0usize; + while unsafe { ffi::fetch::sql_fetch::(stmt) } == SqlReturn::SUCCESS { + for col in 1..=n_cols { + let _ = unsafe { + ffi::fetch::sql_get_data::( + stmt, + col, + CDataType::Default as i16, + buf.as_mut_ptr() as *mut c_void, + buf.len() as isize, + &mut ind, + ) + }; + count += 1; + } + } + count +} + +/// One bound column: keeps its own scratch buffer and indicator alive across fetches. +struct BoundColumn { + buf: Vec, + ind: isize, +} + +impl BoundColumn { + fn new() -> Self { + Self { + buf: vec![0u8; 4096], + ind: 0, + } + } +} + +/// Bind every column once; returns the binding storage. Caller must keep the +/// returned Vec alive for as long as the bindings are in effect. +unsafe fn bind_columns(stmt: *mut c_void, n_cols: u16, c_type: CDataType) -> Vec { + unsafe { + let mut bindings: Vec = (0..n_cols).map(|_| BoundColumn::new()).collect(); + for (i, b) in bindings.iter_mut().enumerate() { + let ret = ffi::bind::sql_bind_col::( + stmt, + (i + 1) as u16, + c_type as i16, + b.buf.as_mut_ptr() as *mut c_void, + b.buf.len() as isize, + &mut b.ind, + ); + assert_eq!(ret, SqlReturn::SUCCESS, "bind_col col {}", i + 1); + } + bindings + } +} + +/// Drain a result set whose columns have already been bound via `bind_columns`. +/// Returns total cells fetched (n_cols × n_rows). +unsafe fn drain_bound_columns(stmt: *mut c_void, n_cols: u16) -> usize { + unsafe { + let mut count = 0usize; + while ffi::fetch::sql_fetch::(stmt) == SqlReturn::SUCCESS { + count += n_cols as usize; + } + count + } +} + +/// Same as `drain_late_binding` but call SQLGetData `repeats` times per cell. +unsafe fn drain_repeat_get_data(stmt: *mut c_void, n_cols: u16, repeats: usize) -> usize { + unsafe { + let mut buf = vec![0u8; 4096]; + let mut ind: isize = 0; + let mut count = 0usize; + while ffi::fetch::sql_fetch::(stmt) == SqlReturn::SUCCESS { + for col in 1..=n_cols { + for _ in 0..repeats { + let _ = ffi::fetch::sql_get_data::( + stmt, + col, + CDataType::Default as i16, + buf.as_mut_ptr() as *mut c_void, + buf.len() as isize, + &mut ind, + ); + count += 1; + } + } + } + count + } +} + +fn bench_shape_a_late_binding(c: &mut Criterion) { + let cfg = bench_config(); + let label = format!("{}x{}", cfg.rows, cfg.cols); + let setup_sql = shape_a_setup_sql(cfg.rows, cfg.cols); + let select_sql = shape_a_select(cfg.cols); + + // Set up the database once, outside the timed block. + let (env, conn, stmt) = unsafe { alloc_handles() }; + assert_eq!(unsafe { connect_memory(conn) }, SqlReturn::SUCCESS); + unsafe { + exec_setup(conn, &setup_sql); + } + + let mut group = c.benchmark_group("sqlite/shape_a"); + group.throughput(Throughput::Elements((cfg.rows * cfg.cols) as u64)); + group.bench_function(BenchmarkId::new("late_binding", &label), |b| { + b.iter_batched( + || unsafe { + // Per-iter: close previous cursor and re-execute SELECT to get a fresh cursor. + let _ = ffi::cursor::sql_close_cursor::(stmt); + assert_eq!(exec_direct(stmt, &select_sql), SqlReturn::SUCCESS); + }, + |_| { + black_box(unsafe { drain_late_binding(stmt, cfg.cols as u16) }); + }, + BatchSize::PerIteration, + ); + }); + group.finish(); + + unsafe { + cleanup(env, conn, stmt); + } +} + +fn bench_shape_b_late_binding(c: &mut Criterion) { + let cfg = bench_config(); + let label = format!("{}x5_len{}", cfg.wide_rows, cfg.wide_str_len); + let setup_sql = shape_b_setup_sql(cfg.wide_rows, cfg.wide_str_len); + let select_sql = shape_b_select(); + + let (env, conn, stmt) = unsafe { alloc_handles() }; + assert_eq!(unsafe { connect_memory(conn) }, SqlReturn::SUCCESS); + unsafe { + exec_setup(conn, &setup_sql); + } + + let mut group = c.benchmark_group("sqlite/shape_b"); + group.throughput(Throughput::Elements((cfg.wide_rows * 5) as u64)); + group.bench_function(BenchmarkId::new("late_binding", &label), |b| { + b.iter_batched( + || unsafe { + let _ = ffi::cursor::sql_close_cursor::(stmt); + assert_eq!(exec_direct(stmt, &select_sql), SqlReturn::SUCCESS); + }, + |_| { + black_box(unsafe { drain_late_binding(stmt, 5) }); + }, + BatchSize::PerIteration, + ); + }); + group.finish(); + + unsafe { + cleanup(env, conn, stmt); + } +} + +fn bench_shape_a_bound(c: &mut Criterion) { + let cfg = bench_config(); + let label = format!("{}x{}", cfg.rows, cfg.cols); + let setup_sql = shape_a_setup_sql(cfg.rows, cfg.cols); + let select_sql = shape_a_select(cfg.cols); + + let (env, conn, stmt) = unsafe { alloc_handles() }; + assert_eq!(unsafe { connect_memory(conn) }, SqlReturn::SUCCESS); + unsafe { + exec_setup(conn, &setup_sql); + } + + // Bind columns once, before the bench loop. Bindings survive SQLCloseCursor + // per ODBC spec, so they remain in effect across all iterations. + assert_eq!( + unsafe { exec_direct(stmt, &select_sql) }, + SqlReturn::SUCCESS + ); + let _bindings = unsafe { bind_columns(stmt, cfg.cols as u16, CDataType::Default) }; + let _ = unsafe { ffi::cursor::sql_close_cursor::(stmt) }; + + let mut group = c.benchmark_group("sqlite/shape_a"); + group.throughput(Throughput::Elements((cfg.rows * cfg.cols) as u64)); + group.bench_function(BenchmarkId::new("bound_columns", &label), |b| { + b.iter_batched( + || unsafe { + let _ = ffi::cursor::sql_close_cursor::(stmt); + assert_eq!(exec_direct(stmt, &select_sql), SqlReturn::SUCCESS); + }, + |_| { + black_box(unsafe { drain_bound_columns(stmt, cfg.cols as u16) }); + }, + BatchSize::PerIteration, + ); + }); + group.finish(); + + unsafe { + cleanup(env, conn, stmt); + } +} + +fn bench_shape_b_bound(c: &mut Criterion) { + let cfg = bench_config(); + let label = format!("{}x5_len{}", cfg.wide_rows, cfg.wide_str_len); + let setup_sql = shape_b_setup_sql(cfg.wide_rows, cfg.wide_str_len); + let select_sql = shape_b_select(); + + let (env, conn, stmt) = unsafe { alloc_handles() }; + assert_eq!(unsafe { connect_memory(conn) }, SqlReturn::SUCCESS); + unsafe { + exec_setup(conn, &setup_sql); + } + + // Bind columns once, before the bench loop. Bindings survive SQLCloseCursor + // per ODBC spec, so they remain in effect across all iterations. + assert_eq!( + unsafe { exec_direct(stmt, &select_sql) }, + SqlReturn::SUCCESS + ); + // Use WChar for the wide-string shape so the bound buffer goes through the UTF-16 path. + let _bindings = unsafe { bind_columns(stmt, 5, CDataType::WChar) }; + let _ = unsafe { ffi::cursor::sql_close_cursor::(stmt) }; + + let mut group = c.benchmark_group("sqlite/shape_b"); + group.throughput(Throughput::Elements((cfg.wide_rows * 5) as u64)); + group.bench_function(BenchmarkId::new("bound_columns", &label), |b| { + b.iter_batched( + || unsafe { + let _ = ffi::cursor::sql_close_cursor::(stmt); + assert_eq!(exec_direct(stmt, &select_sql), SqlReturn::SUCCESS); + }, + |_| { + black_box(unsafe { drain_bound_columns(stmt, 5) }); + }, + BatchSize::PerIteration, + ); + }); + group.finish(); + + unsafe { + cleanup(env, conn, stmt); + } +} + +fn bench_shape_a_repeat(c: &mut Criterion) { + let cfg = bench_config(); + let label = format!("{}x{}", cfg.rows, cfg.cols); + let setup_sql = shape_a_setup_sql(cfg.rows, cfg.cols); + let select_sql = shape_a_select(cfg.cols); + + let (env, conn, stmt) = unsafe { alloc_handles() }; + assert_eq!(unsafe { connect_memory(conn) }, SqlReturn::SUCCESS); + unsafe { + exec_setup(conn, &setup_sql); + } + + let mut group = c.benchmark_group("sqlite/shape_a"); + group.throughput(Throughput::Elements((cfg.rows * cfg.cols) as u64)); + let bench_id = BenchmarkId::new(format!("repeat_get_data_x{}", cfg.repeat_get_data), &label); + group.bench_function(bench_id, |b| { + b.iter_batched( + || unsafe { + let _ = ffi::cursor::sql_close_cursor::(stmt); + assert_eq!(exec_direct(stmt, &select_sql), SqlReturn::SUCCESS); + }, + |_| { + black_box(unsafe { + drain_repeat_get_data(stmt, cfg.cols as u16, cfg.repeat_get_data) + }); + }, + BatchSize::PerIteration, + ); + }); + group.finish(); + + unsafe { + cleanup(env, conn, stmt); + } +} + +fn bench_shape_b_repeat(c: &mut Criterion) { + let cfg = bench_config(); + let label = format!("{}x5_len{}", cfg.wide_rows, cfg.wide_str_len); + let setup_sql = shape_b_setup_sql(cfg.wide_rows, cfg.wide_str_len); + let select_sql = shape_b_select(); + + let (env, conn, stmt) = unsafe { alloc_handles() }; + assert_eq!(unsafe { connect_memory(conn) }, SqlReturn::SUCCESS); + unsafe { + exec_setup(conn, &setup_sql); + } + + let mut group = c.benchmark_group("sqlite/shape_b"); + group.throughput(Throughput::Elements((cfg.wide_rows * 5) as u64)); + let bench_id = BenchmarkId::new(format!("repeat_get_data_x{}", cfg.repeat_get_data), &label); + group.bench_function(bench_id, |b| { + b.iter_batched( + || unsafe { + let _ = ffi::cursor::sql_close_cursor::(stmt); + assert_eq!(exec_direct(stmt, &select_sql), SqlReturn::SUCCESS); + }, + |_| { + black_box(unsafe { drain_repeat_get_data(stmt, 5, cfg.repeat_get_data) }); + }, + BatchSize::PerIteration, + ); + }); + group.finish(); + + unsafe { + cleanup(env, conn, stmt); + } +} + +fn benches() -> Criterion { + configure_for_size(Criterion::default(), bench_config().rows) +} + +criterion_group! { + name = benches_group; + config = benches(); + targets = + bench_shape_a_late_binding, + bench_shape_b_late_binding, + bench_shape_a_bound, + bench_shape_b_bound, + bench_shape_a_repeat, + bench_shape_b_repeat +} +criterion_main!(benches_group); diff --git a/build.rs b/build.rs new file mode 100644 index 0000000..f5b3d1b --- /dev/null +++ b/build.rs @@ -0,0 +1,201 @@ +//! Embeds a Windows `VERSIONINFO` resource in the driver DLL. +//! +//! The ODBC Data Source Administrator reads its **Version** and **Company** +//! columns from the driver file's version resource, and prints `Not marked` +//! for a file that carries none. Every Rust `cdylib` carries none, since rustc +//! emits no such resource. Measured on Windows Server 2022: `sqlsrv32.dll` +//! lists as `10.00.20348.01` / `Microsoft Corporation` and carries exactly +//! those two strings. +//! +//! Nothing here is hand-maintained. Every string comes from `Cargo.toml` +//! through cargo's own environment, so the resource cannot disagree with the +//! package. The version follows `CARGO_PKG_VERSION`, and therefore whatever +//! `cargo-release` wrote into `Cargo.toml`, which is why `release.toml` needs +//! no rule for this file. + +use std::path::PathBuf; +use std::process::Command; + +fn main() { + println!("cargo:rerun-if-changed=build.rs"); + println!("cargo:rerun-if-env-changed=CARGO_PKG_VERSION"); + println!("cargo:rerun-if-env-changed=WINDRES"); + + // `CARGO_CFG_TARGET_OS`, never `cfg!(windows)`: a build script is compiled + // for and run on the *host*, so `cfg!(windows)` describes the machine doing + // the building. The release DLL is cross-compiled from Linux, where it is + // false, so it would skip the resource on precisely the build that ships. + if std::env::var("CARGO_CFG_TARGET_OS").as_deref() != Ok("windows") { + return; + } + + // Only the GNU toolchain is wired up. An MSVC target needs `rc.exe` from a + // Visual Studio installation, which this repo never builds with: + // `.github/workflows/release.yaml` installs `gcc-mingw-w64-x86-64` and + // builds `x86_64-pc-windows-gnu`. Warn rather than fail, so a local MSVC + // build still works. It produces a DLL the Administrator lists as + // `Not marked`. + let target_env = std::env::var("CARGO_CFG_TARGET_ENV").unwrap_or_default(); + if target_env != "gnu" { + println!( + "cargo:warning=no version resource embedded: the {target_env} Windows toolchain \ + needs rc.exe, and only the gnu toolchain's windres is wired up in build.rs. \ + The ODBC Data Source Administrator will list this driver as \"Not marked\"." + ); + return; + } + + let out_dir = PathBuf::from( + std::env::var("OUT_DIR").expect("cargo always sets OUT_DIR for a build script"), + ); + let rc_path = out_dir.join("version.rc"); + let obj_path = out_dir.join("version.o"); + + if let Err(e) = std::fs::write(&rc_path, version_rc()) { + fail(&format!("could not write {}: {e}", rc_path.display())); + } + + let windres = windres_command(); + let status = Command::new(&windres) + .arg("--input") + .arg(&rc_path) + .arg("--output") + .arg(&obj_path) + // COFF, so the result is an object file the linker takes like any + // other. windres defaults to emitting an `.rc` back out. + .arg("--output-format=coff") + .status(); + + match status { + Ok(s) if s.success() => {} + Ok(s) => fail(&format!( + "`{windres}` failed with {s} on {}", + rc_path.display() + )), + Err(e) => fail(&format!( + "could not run `{windres}`: {e}\n\ + A Windows build needs windres to embed the driver's version resource. \ + On Debian and Ubuntu it is in binutils-mingw-w64-x86-64, which \ + gcc-mingw-w64-x86-64 already depends on. Set WINDRES to override the name." + )), + } + + // `-cdylib`, not the unsuffixed form: the resource belongs to the shipped + // DLL, and the unsuffixed flag would also be handed to the linker for every + // test and benchmark binary. + println!("cargo:rustc-link-arg-cdylib={}", obj_path.display()); +} + +/// Abort the build with a message. +/// +/// `exit` rather than `panic!`, which the crate's clippy configuration denies. +/// Cargo renders a build script's stderr and its exit status as a build error +/// either way, and this way adds no backtrace for nobody to read. +fn fail(message: &str) -> ! { + eprintln!("error: {message}"); + std::process::exit(1); +} + +/// The `windres` to invoke. +/// +/// The cross-prefixed name first, because that is what a Linux host has: the +/// bare `windres` there, if it exists at all, targets the host. `WINDRES` +/// overrides both, for a toolchain under a different prefix. +fn windres_command() -> String { + if let Ok(explicit) = std::env::var("WINDRES") { + return explicit; + } + let prefixed = "x86_64-w64-mingw32-windres"; + if Command::new(prefixed).arg("--version").output().is_ok() { + return prefixed.to_string(); + } + "windres".to_string() +} + +/// The resource script, built entirely from cargo's environment. +fn version_rc() -> String { + let version = std::env::var("CARGO_PKG_VERSION").unwrap_or_default(); + let major = env_num("CARGO_PKG_VERSION_MAJOR"); + let minor = env_num("CARGO_PKG_VERSION_MINOR"); + let patch = env_num("CARGO_PKG_VERSION_PATCH"); + let description = std::env::var("CARGO_PKG_DESCRIPTION").unwrap_or_default(); + let license = std::env::var("CARGO_PKG_LICENSE").unwrap_or_default(); + let company = company_name(); + + // The lib name is the package name with hyphens replaced, which is what + // both the `.so` and the `.dll` are named after. + let file_name = format!( + "{}.dll", + std::env::var("CARGO_PKG_NAME") + .unwrap_or_default() + .replace('-', "_") + ); + + // Literal numeric constants rather than `#include `, so the + // script does not depend on the mingw headers being on windres's include + // path: VOS_NT_WINDOWS32 (0x40004) and VFT_DLL (0x2). + // + // The 040904b0 block is US English, Unicode, and the VarFileInfo + // translation below must name the same pair or the strings are ignored. + format!( + r#"1 VERSIONINFO +FILEVERSION {major},{minor},{patch},0 +PRODUCTVERSION {major},{minor},{patch},0 +FILEOS 0x40004L +FILETYPE 0x2L +BEGIN + BLOCK "StringFileInfo" + BEGIN + BLOCK "040904b0" + BEGIN + VALUE "CompanyName", "{company}" + VALUE "FileDescription", "{description}" + VALUE "FileVersion", "{version}" + VALUE "InternalName", "{file_name}" + VALUE "LegalCopyright", "Copyright the {company} authors. Licensed under {license}." + VALUE "OriginalFilename", "{file_name}" + VALUE "ProductName", "{description}" + VALUE "ProductVersion", "{version}" + END + END + BLOCK "VarFileInfo" + BEGIN + VALUE "Translation", 0x409, 1200 + END +END +"#, + company = rc_escape(&company), + description = rc_escape(&description), + license = rc_escape(&license), + version = rc_escape(&version), + file_name = rc_escape(&file_name), + ) +} + +/// `CARGO_PKG_AUTHORS` without the address, so `Cargo.toml`'s +/// `Stackable GmbH ` becomes the company the ODBC +/// Administrator shows. Only the first author: the column holds one name. +fn company_name() -> String { + let authors = std::env::var("CARGO_PKG_AUTHORS").unwrap_or_default(); + let first = authors.split(':').next().unwrap_or_default(); + first + .split('<') + .next() + .unwrap_or_default() + .trim() + .to_string() +} + +fn env_num(key: &str) -> u16 { + std::env::var(key) + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(0) +} + +/// Quote and backslash are the two characters an `.rc` string cannot carry +/// raw. None of the values used here contains either today; escaping them +/// anyway keeps a future `description` from producing an unparseable script. +fn rc_escape(value: &str) -> String { + value.replace('\\', r"\\").replace('"', r#"\""#) +} diff --git a/clippy.toml b/clippy.toml new file mode 100644 index 0000000..f69b4a6 --- /dev/null +++ b/clippy.toml @@ -0,0 +1,2 @@ +allow-unwrap-in-tests = true +allow-panic-in-tests = true diff --git a/deny.toml b/deny.toml new file mode 100644 index 0000000..b67e18c --- /dev/null +++ b/deny.toml @@ -0,0 +1,41 @@ +# Cargo deny configuration for stackable-odbc-sqlite +# Based on operator-rs conventions. +# Run: cargo deny check + +[graph] +targets = [ + { triple = "x86_64-unknown-linux-gnu" }, + { triple = "aarch64-unknown-linux-gnu" }, + { triple = "x86_64-pc-windows-gnu" }, +] + +[advisories] +yanked = "deny" + +[bans] +multiple-versions = "allow" + +[licenses] +unused-allowed-license = "allow" +confidence-threshold = 1.0 +allow = [ + "Apache-2.0", + "BSD-2-Clause", + "BSD-3-Clause", + "CC0-1.0", + "ISC", + "MIT", + "MPL-2.0", + "Unicode-3.0", + "Unicode-DFS-2016", + "Zlib", + "Unlicense", +] +private = { ignore = true } + +[sources] +unknown-registry = "deny" +unknown-git = "deny" +allow-git = [ + "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/stackabletech/stackable-odbc-core.git", +] diff --git a/integration-tests/README.md b/integration-tests/README.md new file mode 100644 index 0000000..0517e98 --- /dev/null +++ b/integration-tests/README.md @@ -0,0 +1,172 @@ +# Integration tests + +Everything that exercises the driver from outside the Rust crate: through real +unixODBC on Linux, and through the real Windows Driver Manager in a VM. + +There is no service to start. SQLite is a file, and `rusqlite` links its own +copy of it into the driver, so the whole suite runs on a bare runner in seconds. +That is exactly why it gates every pull request while the Trino driver's +equivalent cannot. + +```bash +./integration-tests/setup.sh # build the driver, create the database, write the ODBC config +./integration-tests/run-tests.sh # pyodbc through unixODBC, then cargo test +``` + +Both take `--help`. + +## Layout + +| Path | What it holds | +|------|---------------| +| `setup.sh`, `run-tests.sh` | Wrappers. The logic is in `scripts/` | +| `scripts/lib.sh` | The paths and helpers both scripts share. Sourced, never executed | +| `scripts/setup.sh` | Builds the driver, creates `test.db`, writes `odbc.ini` / `odbcinst.ini` | +| `scripts/run-tests.sh` | Runs the suites | +| `suites/create_test_db.sql` | The schema and rows every suite reads | +| `suites/harness.py` | PASS/FAIL accounting and connection-string parsing, shared by the suites | +| `suites/odbc_abi.py` | The raw ODBC C ABI declared for `ctypes`, for the suites that skip the Driver Manager | +| `suites/test_integration.py` | The pyodbc suite, run once per connection style | +| `suites/test_transactions.py` | Manual-commit transactions, run once per connection style | +| `suites/test_sql_surface.py` | The SQL a BI tool emits, the ODBC escapes and the catalog functions, run once per connection style | +| `suites/test_c_abi.py` | The C ABI pen test, run once | +| `suites/test_type_matrix.py` | Type-transform fuzz and column description, run once | +| `perf/test_stress.py` | BI query patterns over a generated star schema, run once | +| `generated/` | Everything `setup.sh` writes. Gitignored | +| `windows/` | The VM suite, its libvirt definitions, and [WINDOWS.md](windows/WINDOWS.md) | + +`generated/` is ignored rather than committed because the ODBC config it holds +names absolute paths. `odbcinst.ini` points at the driver's `.so` and +`odbc.ini` at the database, so neither survives being moved to another +checkout, and a committed copy would be wrong for everyone but its author. + +## What gets run + +`run-tests.sh` runs the pyodbc suite **twice**, against the same database: + +- **DSN-less**, `Driver=...;Database=...`, which exercises this driver's own + connection-string parsing. +- **Via a DSN**, `DSN=test_sqlite`, where the Driver Manager resolves the + keywords out of `odbc.ini` first. + +They are separate runs because they fail separately. A driver that reads its +parameters correctly can still be unreachable through a DSN, and that is a +configuration most applications actually use. + +`test_transactions.py` runs the same two ways, because manual-commit mode is +set on the connection. Three of its scenarios assert the *opposite* of the +Trino driver's equivalents, which is why it could not simply be copied across: +a failed statement leaves a SQLite transaction usable rather than aborting it, +a commit preserves an open cursor rather than closing it, and serializable is +the level that must be accepted rather than refused. + +`test_sql_surface.py` runs both ways too. It walks joins, aggregates, window +functions, CTEs, set operations and parameters, and is the only suite that +reaches `escape_dialect.rs`: the `{fn ...}`, `{d ...}`, `{ts ...}` and +`{oj ...}` sequences, including the three date/time forms that are bare +keywords in SQLite and need the whole escape rewritten rather than the name +swapped. Where the Trino driver can only check that a key or index lookup +returns nothing without erroring, this one asserts the rows, because SQLite +publishes all three. + +It also asserts the two things that translation getting *too* eager would +break. First, the boundaries: a `{fn ...}` inside a string literal, a comment, +or any of SQLite's three identifier-quoting styles has to survive verbatim, +because rewriting there changes the value a query returns with no error +anywhere. Second, the bitmaps as a contract: every bit set in +`SQL_STRING_FUNCTIONS` and its three siblings is read back from `SQLGetInfo` +and the matching `{fn NAME(...)}` executed with spec-shaped arguments. A BI +tool emits an escape only for the bits the driver sets, so a set bit that does +not execute is a query the tool will build and the driver will reject, and +nothing else ties `info.rs`'s bitmaps to `escape_dialect.rs`'s remap table. + +Last in that suite, the catalog functions are given hostile names. They are the +only path in the driver that turns a caller-supplied argument into SQL, and for +a BI tool that argument is often typed into a filter box. A table whose name +contains a quote has to be found, payloads that close a literal and issue a +`DROP` have to be treated as names that match nothing (asserted by re-counting +the fixture afterwards, since "no exception" would also pass for a driver that +ran them), and `%` and `_` have to keep working as patterns, which rules out +escaping everything indiscriminately. + +Then `test_c_abi.py`, once. It loads the driver's `.so` with `ctypes` and calls +the exported entry points with **no Driver Manager in the loop**, which is the +point: unixODBC answers a large part of the ODBC state machine itself, so what +the driver does with an out-of-order or malformed call is invisible to anything +going through pyodbc. It covers handle lifecycle and parentage, stale handles +and double frees, cursor state, attribute round-trips, the query timeout, and +transactions. It also covers the `SQLGetData` buffer contract, which is the +part of that call an application cannot avoid and a Driver Manager does not +implement: how much is written, what the indicator counts, that a zero-length +call is the documented length probe rather than a completed read, that a +second call continues the value instead of restarting it (a driver that +restarts turns the documented drain loop into an infinite one), and that an +ordinal past the last column is `07009` rather than a general error. A DSN run +would reach the same code by a longer route, so there is only one. + +Because the spec's **(DM)** diagnostics come from the Driver Manager, that suite +never demands one. Where a SQLSTATE is (DM)-annotated it asserts what the driver +does instead, with a comment naming the diagnostic it is not asking for. + +Last, `test_type_matrix.py`, also once and also through ctypes. It drives every +(value, C type) pair through `SQLGetData` and checks invariants rather than a +transcribed copy of the ODBC conversion matrix, which would mostly test the +transcription. It then checks what `SQLDescribeCol` *says* each column is, which +is a separate question from what `SQLGetData` will hand over: SQLite gives a +computed column no declared type, so the driver answers from the storage class +of the values, and a tool decides from that whether a column can be summed. + +Last, `perf/test_stress.py`, which asks the same shapes of SQL at size: +multi-table joins over a generated star schema, 50,000-row fetches, wide rows +and a prepared statement re-executed two hundred times. That matters more here +than for a client-server driver, because `exec_direct` materialises every row +before returning, so a large result set is where that decision is felt. It +builds its fixture with recursive CTEs and works in a database of its own, +`generated/stress.db`, so the shared one keeps the size the other suites +expect. + +There is no counterpart to the Trino driver's `perf/parse_profile.py` and +`perf/profile_stress.sh`. Those split a query's time between the coordinator +and the client, and SQLite has no server-side half to attribute anything to. +`cargo bench` measures fetch throughput instead. + +It then runs `cargo test`, so that one command gives a developer the whole +suite. CI passes `--skip-cargo-test`, since its pre-commit job has already run +exactly that via the `cargo-test` hook. + +## Options + +| Flag | Effect | +|------|--------| +| `--skip-build` | Reuse the driver already built. Forwarded to `windows_test.py`, whose build is a separate cross-compile | +| `--skip-cargo-test` | Run the pyodbc suites only. What CI passes | +| `--windows` | Additionally run the suite inside the Windows VM | + +Any other argument is forwarded to `windows_test.py` (`--target`, `--host`, +`--vm-network`, `--user`, `--password`, `--gateway`) and so is rejected without +`--windows`, since a flag forwarded to a script that never runs would be +silently ignored. + +## Windows + +`windows/windows_test.py` deploys the cross-compiled DLL to a provisioned +libvirt VM over WinRM, registers it, and runs the same suites through the +Windows Driver Manager: the three pyodbc ones DSN-less and then via a DSN, the +two ctypes ones once each, and the stress suite once. The Windows DM is much +stricter than unixODBC and tends to fail silently, so this is measured rather +than assumed. + +See [windows/WINDOWS.md](windows/WINDOWS.md) for provisioning the VM. + +## Interactively + +`setup.sh` prints these at the end, with absolute paths filled in: + +```bash +export ODBCSYSINI=$(pwd)/integration-tests/generated +export ODBCINI=$(pwd)/integration-tests/generated/odbc.ini +isql -3 test_sqlite -v +``` + +Prefix either with `ODBC_LOG_LEVEL=debug` to see which ODBC functions your +client calls, and in what order. diff --git a/integration-tests/generated/.gitignore b/integration-tests/generated/.gitignore new file mode 100644 index 0000000..df428c9 --- /dev/null +++ b/integration-tests/generated/.gitignore @@ -0,0 +1,4 @@ +# The ODBC config setup.sh writes here embeds absolute paths, so none of it is +# portable between checkouts. Keep the directory, ignore the contents. +* +!.gitignore diff --git a/integration-tests/perf/test_stress.py b/integration-tests/perf/test_stress.py new file mode 100644 index 0000000..0fbdfc0 --- /dev/null +++ b/integration-tests/perf/test_stress.py @@ -0,0 +1,398 @@ +#!/usr/bin/env python3 +""" +BI stress tests for the SQLite ODBC driver. + +Exercises the query patterns a BI tool emits, at a size the rest of the suite +does not reach: multi-table joins over a star schema, UNIONs, subqueries, CTEs, +window functions, large result sets and wide rows. + +Where `test_sql_surface.py` asks whether a shape of SQL works at all, this asks +whether it still works over tens of thousands of rows. That matters more here +than for a client-server driver, because `exec_direct` materialises every row +before returning: a result set is held in memory in full, so a large one is the +case where that decision is felt. See the eager-materialisation section of +AGENTS.md. + +The fixture is generated by recursive CTE rather than shipped, so the suite +carries no data file and the size is a constant at the top of this module. + +There is no counterpart to the Trino driver's `parse_profile.py` and +`profile_stress.sh`. Those attribute a query's time between the coordinator and +the client, and SQLite has no server-side half: the query runs in-process, +inside the same shared object. `cargo bench` (`benches/fetch_sqlite.rs`) is +where fetch throughput is measured instead. + +Usage: + python3 integration-tests/perf/test_stress.py \ + "Driver=/path/to/libstackable_odbc_sqlite.so;Database=/path/to/test.db" + python3 integration-tests/perf/test_stress.py "DSN=test_sqlite" + +Needs no server. Requires `pyodbc`, normally through `uv run --with pyodbc`. +""" + +import os +import sys + +import pyodbc + +sys.path.insert( + 0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "suites") +) + +from harness import Results, Target # noqa: E402 + +R = Results("stress") + +CUSTOMERS = 5_000 +ITEMS = 500 +SALES = 50_000 +# Customers above this have no sales, so a LEFT JOIN has NULLs to produce. +CUSTOMERS_WITH_SALES = 4_000 + +CUSTOMER = "stress_customer" +ITEM = "stress_item" +SALES_TABLE = "stress_sales" + + +def make_fixture(cur): + """A small star schema, generated in three statements. + + Sales are dropped first and inserted last: the driver turns foreign-key + enforcement on for every connection, so the order is not optional. + """ + cur.execute(f"DROP TABLE IF EXISTS {SALES_TABLE}") + cur.execute(f"DROP TABLE IF EXISTS {CUSTOMER}") + cur.execute(f"DROP TABLE IF EXISTS {ITEM}") + + cur.execute( + f"CREATE TABLE {CUSTOMER} (" + " id INTEGER PRIMARY KEY," + " first_name TEXT," + " last_name TEXT," + " region TEXT" + ")" + ) + cur.execute( + f"CREATE TABLE {ITEM} (id INTEGER PRIMARY KEY, product_name TEXT, category TEXT)" + ) + cur.execute( + f"CREATE TABLE {SALES_TABLE} (" + " id INTEGER PRIMARY KEY," + f" customer_id INTEGER REFERENCES {CUSTOMER}(id)," + f" item_id INTEGER REFERENCES {ITEM}(id)," + " channel TEXT," + " quantity INTEGER," + " net_paid REAL" + ")" + ) + + # Every tenth customer has a NULL last name, so the NULL probes have + # something to find in a column that is not the join key. + cur.execute( + f"INSERT INTO {CUSTOMER} (id, first_name, last_name, region) " + "WITH RECURSIVE seq(i) AS (SELECT 1 UNION ALL SELECT i + 1 FROM seq " + f"WHERE i < {CUSTOMERS}) " + "SELECT i, 'first' || i, " + "CASE WHEN i % 10 = 0 THEN NULL ELSE 'last' || i END, " + "'region' || (i % 10) FROM seq" + ) + cur.execute( + f"INSERT INTO {ITEM} (id, product_name, category) " + "WITH RECURSIVE seq(i) AS (SELECT 1 UNION ALL SELECT i + 1 FROM seq " + f"WHERE i < {ITEMS}) " + "SELECT i, 'product' || i, 'category' || (i % 20) FROM seq" + ) + cur.execute( + f"INSERT INTO {SALES_TABLE} " + "(id, customer_id, item_id, channel, quantity, net_paid) " + "WITH RECURSIVE seq(i) AS (SELECT 1 UNION ALL SELECT i + 1 FROM seq " + f"WHERE i < {SALES}) " + f"SELECT i, (i % {CUSTOMERS_WITH_SALES}) + 1, (i % {ITEMS}) + 1, " + "CASE WHEN i % 2 = 0 THEN 'store' ELSE 'web' END, " + "(i % 7) + 1, " + # Every hundredth sale has no amount, so the aggregates meet a NULL. + "CASE WHEN i % 100 = 0 THEN NULL ELSE ((i % 1000) + 1) * 1.5 END " + "FROM seq" + ) + + +def drop_fixture(cur): + for table in (SALES_TABLE, CUSTOMER, ITEM): + try: + cur.execute(f"DROP TABLE IF EXISTS {table}") + except Exception: # noqa: BLE001 + pass + + +def main(): + target = Target.from_argv( + sys.argv, + "usage: test_stress.py " + '"Driver=/path/to/libstackable_odbc_sqlite.so;Database=/path/to/test.db"', + ) + conn = pyodbc.connect(target.conn_str(), autocommit=True) + cur = conn.cursor() + + print(f"building the fixture: {CUSTOMERS} customers, {ITEMS} items, {SALES} sales") + make_fixture(cur) + + try: + # -------------------------------------------------------------- + # Multi-table joins + # -------------------------------------------------------------- + def two_table_join(): + cur.execute(f""" + SELECT c.first_name, c.last_name, SUM(s.net_paid) AS total_spend + FROM {CUSTOMER} c + JOIN {SALES_TABLE} s ON c.id = s.customer_id + GROUP BY c.first_name, c.last_name + ORDER BY total_spend DESC + LIMIT 10 + """) + rows = cur.fetchall() + assert len(rows) == 10, f"expected 10 rows, got {len(rows)}" + for row in rows: + assert row[2] is not None and row[2] > 0, f"total_spend was {row[2]!r}" + + R.run("Two-table INNER JOIN with aggregation", two_table_join) + + def three_table_star_join(): + cur.execute(f""" + SELECT c.first_name, i.product_name, SUM(s.quantity) AS total_qty + FROM {SALES_TABLE} s + JOIN {CUSTOMER} c ON s.customer_id = c.id + JOIN {ITEM} i ON s.item_id = i.id + GROUP BY c.first_name, i.product_name + ORDER BY total_qty DESC + LIMIT 10 + """) + rows = cur.fetchall() + assert len(rows) == 10, f"expected 10 rows, got {len(rows)}" + for row in rows: + assert row[2] is not None, "total_qty should not be NULL" + + R.run("Three-table star-schema JOIN", three_table_star_join) + + def left_join_nulls(): + cur.execute(f""" + SELECT c.id, c.first_name, s.id + FROM {CUSTOMER} c + LEFT JOIN {SALES_TABLE} s ON c.id = s.customer_id + WHERE c.id > {CUSTOMERS_WITH_SALES} + ORDER BY c.id + LIMIT 50 + """) + rows = cur.fetchall() + assert len(rows) == 50, f"expected 50 rows, got {len(rows)}" + assert all(row[2] is None for row in rows), ( + "every customer past the sales range must join to NULL" + ) + + R.run("LEFT JOIN producing NULLs", left_join_nulls) + + # -------------------------------------------------------------- + # Subqueries and CTEs + # -------------------------------------------------------------- + def in_subquery(): + cur.execute(f""" + SELECT id, first_name FROM {CUSTOMER} + WHERE id IN ( + SELECT customer_id FROM {SALES_TABLE} WHERE net_paid > 1000 + ) + ORDER BY id + LIMIT 10 + """) + rows = cur.fetchall() + assert len(rows) == 10, f"expected 10 rows, got {len(rows)}" + for row in rows: + assert row[0] > 0, f"id should be > 0, got {row[0]}" + + R.run("IN subquery over the fact table", in_subquery) + + def correlated_subquery(): + cur.execute(f""" + SELECT c.id, + (SELECT COUNT(*) FROM {SALES_TABLE} s WHERE s.customer_id = c.id) AS n + FROM {CUSTOMER} c + WHERE c.id <= 20 + ORDER BY c.id + """) + rows = cur.fetchall() + assert len(rows) == 20, f"expected 20 rows, got {len(rows)}" + assert all(row[1] > 0 for row in rows), "each of these customers has sales" + + R.run("Correlated subquery", correlated_subquery) + + def cte(): + cur.execute(f""" + WITH per_region AS ( + SELECT c.region, SUM(s.net_paid) AS spend + FROM {CUSTOMER} c + JOIN {SALES_TABLE} s ON c.id = s.customer_id + GROUP BY c.region + ) + SELECT region, spend FROM per_region ORDER BY spend DESC + """) + rows = cur.fetchall() + assert len(rows) == 10, f"expected 10 regions, got {len(rows)}" + + R.run("CTE / WITH clause", cte) + + # -------------------------------------------------------------- + # Set operations + # -------------------------------------------------------------- + def union_all(): + # Each branch is wrapped in a subquery because SQLite refuses a + # LIMIT on an operand of a compound SELECT ("LIMIT clause should + # come after UNION ALL not before"), where Trino accepts it. The + # LIMIT has to bind per branch here, not to the union. + cur.execute(f""" + SELECT customer_id FROM ( + SELECT customer_id FROM {SALES_TABLE} + WHERE channel = 'store' LIMIT 1000 + ) + UNION ALL + SELECT customer_id FROM ( + SELECT customer_id FROM {SALES_TABLE} + WHERE channel = 'web' LIMIT 1000 + ) + """) + rows = cur.fetchall() + assert len(rows) == 2000, f"expected 2000 rows, got {len(rows)}" + + R.run("UNION ALL across sales channels", union_all) + + def union_dedup(): + cur.execute(f""" + SELECT region FROM {CUSTOMER} + UNION + SELECT region FROM {CUSTOMER} + """) + rows = cur.fetchall() + assert len(rows) == 10, f"expected 10 distinct regions, got {len(rows)}" + + R.run("UNION with dedup", union_dedup) + + # -------------------------------------------------------------- + # Result-set shapes + # -------------------------------------------------------------- + def large_result_set(): + cur.execute(f"SELECT id, customer_id, net_paid FROM {SALES_TABLE} ORDER BY id") + rows = cur.fetchall() + assert len(rows) == SALES, f"expected {SALES} rows, got {len(rows)}" + # Ordered, so a dropped or duplicated row shows up as a gap rather + # than only as a count that happens to match. + assert rows[0][0] == 1 and rows[-1][0] == SALES, ( + f"first and last ids were {rows[0][0]} and {rows[-1][0]}" + ) + + R.run(f"Fetch all {SALES} rows", large_result_set) + + def wide_result_set(): + cur.execute(f""" + SELECT s.id, s.customer_id, s.item_id, s.channel, s.quantity, s.net_paid, + c.first_name, c.last_name, c.region, i.product_name, i.category + FROM {SALES_TABLE} s + JOIN {CUSTOMER} c ON s.customer_id = c.id + JOIN {ITEM} i ON s.item_id = i.id + LIMIT 5000 + """) + assert len(cur.description) == 11, ( + f"expected 11 columns, got {len(cur.description)}" + ) + rows = cur.fetchall() + assert len(rows) == 5000, f"expected 5000 rows, got {len(rows)}" + assert len(rows[0]) == 11, f"expected 11 values, got {len(rows[0])}" + + R.run("Wide result set (11 columns)", wide_result_set) + + def nulls_in_various_positions(): + cur.execute(f""" + SELECT id, last_name, region FROM {CUSTOMER} + WHERE id % 10 = 0 + ORDER BY id + LIMIT 100 + """) + rows = cur.fetchall() + assert len(rows) == 100, f"expected 100 rows, got {len(rows)}" + assert all(row[1] is None for row in rows), ( + "every tenth customer has a NULL last name" + ) + assert all(row[0] is not None and row[2] is not None for row in rows), ( + "the columns either side of the NULL must survive it" + ) + + R.run("NULLs in various column positions", nulls_in_various_positions) + + def empty_result_set(): + cur.execute(f"SELECT id FROM {CUSTOMER} WHERE id < 0") + rows = cur.fetchall() + assert rows == [], f"expected no rows, got {len(rows)}" + # The describe still has to work: an application sizes its buffers + # from it before it knows the set is empty. + assert cur.description is not None, "an empty result set still has columns" + + R.run("Empty result set", empty_result_set) + + # -------------------------------------------------------------- + # Aggregation and windows at size + # -------------------------------------------------------------- + def group_by_having_on_a_join(): + cur.execute(f""" + SELECT c.region, COUNT(*) AS n, SUM(s.net_paid) AS spend + FROM {CUSTOMER} c + JOIN {SALES_TABLE} s ON c.id = s.customer_id + GROUP BY c.region + HAVING COUNT(*) > 100 + ORDER BY spend DESC + """) + rows = cur.fetchall() + assert len(rows) == 10, f"expected 10 regions, got {len(rows)}" + assert all(row[1] > 100 for row in rows), "HAVING was not applied" + + R.run("GROUP BY + HAVING on a JOIN", group_by_having_on_a_join) + + def window_function(): + cur.execute(f""" + SELECT id, customer_id, rn FROM ( + SELECT s.id, s.customer_id, + ROW_NUMBER() OVER (PARTITION BY s.customer_id ORDER BY s.id) AS rn + FROM {SALES_TABLE} s + WHERE s.customer_id <= 100 + ) + WHERE rn = 1 + ORDER BY customer_id + """) + rows = cur.fetchall() + assert len(rows) == 100, f"expected 100 partitions, got {len(rows)}" + assert all(row[2] == 1 for row in rows), "every row must be its partition's first" + + R.run("Window function (ROW_NUMBER OVER PARTITION BY)", window_function) + + # -------------------------------------------------------------- + # The parameter path, repeated + # -------------------------------------------------------------- + def prepared_statement_reused(): + """A prepared statement re-executed many times. + + This is what a BI tool does when it pages through a dimension, and + it is the path where a leak or a stale cursor would accumulate + rather than show up once. + """ + sql = f"SELECT COUNT(*) FROM {SALES_TABLE} WHERE customer_id = ?" + total = 0 + for customer_id in range(1, 201): + total += cur.execute(sql, [customer_id]).fetchone()[0] + assert total > 0, "the repeated query found nothing" + + R.run("Prepared statement re-executed 200 times", prepared_statement_reused) + finally: + drop_fixture(cur) + cur.close() + conn.close() + + return R.summary() + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/integration-tests/run-tests.sh b/integration-tests/run-tests.sh new file mode 100755 index 0000000..308e84d --- /dev/null +++ b/integration-tests/run-tests.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +# Wrapper. The logic lives in scripts/run-tests.sh. +exec "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/scripts/run-tests.sh" "$@" diff --git a/integration-tests/scripts/lib.sh b/integration-tests/scripts/lib.sh new file mode 100644 index 0000000..3761819 --- /dev/null +++ b/integration-tests/scripts/lib.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# Shared paths and helpers. Sourced, never executed. +# +# SC2034: every variable below is consumed by a script that sources this file, +# which shellcheck cannot see from here. +# shellcheck disable=SC2034 + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +TEST_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +PROJECT_DIR="$(cd "$TEST_DIR/.." && pwd)" + +SUITES_DIR="$TEST_DIR/suites" +PERF_DIR="$TEST_DIR/perf" +WINDOWS_DIR="$TEST_DIR/windows" + +# Everything setup.sh writes lands here, and the whole directory is gitignored. +# odbc.ini names the database and odbcinst.ini names the driver library, both by +# absolute path, so neither survives being moved to another checkout. +GENERATED="$TEST_DIR/generated" +DB_PATH="$GENERATED/test.db" +# The stress suite generates 50k rows. It gets a database of its own so the +# shared one keeps the size the other suites expect, and so a stress run left +# half-finished cannot affect them. SQLite creates the file on first connect. +STRESS_DB_PATH="$GENERATED/stress.db" +ODBC_INI="$GENERATED/odbc.ini" +ODBCINST_INI="$GENERATED/odbcinst.ini" + +DRIVER_PATH="$PROJECT_DIR/target/debug/libstackable_odbc_sqlite.so" + +# The DSN setup.sh writes into odbc.ini, and the one run-tests.sh connects +# through for its second configuration. +DSN_NAME="test_sqlite" + +mkdir -p "$GENERATED" + +# Point unixODBC at the generated configuration rather than the system's. +# ODBCSYSINI is a *directory* (unixODBC appends `odbcinst.ini` itself) while +# ODBCINI is a full path, which is why the two are not spelled alike. +use_generated_odbc_config() { + export ODBCSYSINI="$GENERATED" + export ODBCINI="$ODBC_INI" +} + +# Build the cdylib pyodbc loads. `cargo test` builds the test harness, not this, +# so a run that skips it would silently exercise the previous build. cargo is +# incremental, so repeating it costs nothing when nothing changed. +build_driver() { + echo "=== Building stackable-odbc-sqlite ===" + (cd "$PROJECT_DIR" && cargo build) +} + +# usage