diff --git a/.github/workflows/ExtensionTemplate.yml b/.github/workflows/ExtensionTemplate.yml deleted file mode 100644 index 05723d4..0000000 --- a/.github/workflows/ExtensionTemplate.yml +++ /dev/null @@ -1,162 +0,0 @@ -# -# NOTE: this workflow is for testing the extension template itself, -# this workflow will be removed when scripts/bootstrap-template.py is run -# -name: Extension Template -on: [push, pull_request,repository_dispatch] -concurrency: - group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }} - cancel-in-progress: true - -jobs: - linux: - name: Linux - if: ${{ vars.RUN_RENAME_TEST == 'true' || github.repository == 'duckdb/extension-template' }} - runs-on: ubuntu-latest - strategy: - matrix: - # Add commits/tags to build against other DuckDB versions - duckdb_version: [ '' ] - env: - VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake - VCPKG_TARGET_TRIPLET: 'x64-linux' - GEN: ninja - ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true - defaults: - run: - shell: bash - - steps: - - name: Install Ninja - shell: bash - run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build - - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - submodules: 'true' - - - name: Checkout DuckDB to version - if: ${{ matrix.duckdb_version != ''}} - run: | - cd duckdb - git checkout ${{ matrix.duckdb_version }} - - - name: Setup vcpkg - uses: lukka/run-vcpkg@v11.1 - with: - vcpkgGitCommitId: ce613c41372b23b1f51333815feb3edd87ef8a8b - - - name: Rename extension - run: | - python3 scripts/bootstrap-template.py ext_1_a_123b_b11 - - - name: Build - run: | - make - - - name: Test - run: | - make test - - macos: - name: MacOS - if: ${{ vars.RUN_RENAME_TEST == 'true' || github.repository == 'duckdb/extension-template' }} - runs-on: macos-latest - strategy: - matrix: - # Add commits/tags to build against other DuckDB versions - duckdb_version: [ ''] - env: - VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake - VCPKG_TARGET_TRIPLET: 'x64-osx' - OSX_BUILD_ARCH: 'x86_64' - GEN: ninja - defaults: - run: - shell: bash - - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - submodules: 'true' - - - name: Install Ninja - run: brew install ninja - - - uses: actions/setup-python@v2 - with: - python-version: '3.11' - - - name: Checkout DuckDB to version - if: ${{ matrix.duckdb_version != ''}} - run: | - cd duckdb - git checkout ${{ matrix.duckdb_version }} - - - name: Setup vcpkg - uses: lukka/run-vcpkg@v11.1 - with: - vcpkgGitCommitId: ce613c41372b23b1f51333815feb3edd87ef8a8b - - - name: Rename extension - run: | - python scripts/bootstrap-template.py ext_1_a_123b_b11 - - - name: Build - run: | - make - - - name: Test - run: | - make test - - windows: - name: Windows - if: ${{ vars.RUN_RENAME_TEST == 'true' || github.repository == 'duckdb/extension-template' }} - runs-on: windows-latest - strategy: - matrix: - # Add commits/tags to build against other DuckDB versions - duckdb_version: [ '' ] - env: - VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake - VCPKG_TARGET_TRIPLET: 'x64-windows-static-md' - defaults: - run: - shell: bash - - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - submodules: 'true' - - - uses: actions/setup-python@v2 - with: - python-version: '3.11' - - - name: Checkout DuckDB to version - # Add commits/tags to build against other DuckDB versions - if: ${{ matrix.duckdb_version != ''}} - run: | - cd duckdb - git checkout ${{ matrix.duckdb_version }} - - - name: Setup vcpkg - uses: lukka/run-vcpkg@v11.1 - with: - vcpkgGitCommitId: ce613c41372b23b1f51333815feb3edd87ef8a8b - - - name: Rename extension - run: | - python scripts/bootstrap-template.py ext_1_a_123b_b11 - - - name: Build - run: | - make - - - name: Test extension - run: | - build/release/test/Release/unittest.exe diff --git a/.github/workflows/MainDistributionPipeline.yml b/.github/workflows/MainDistributionPipeline.yml index a016aa5..6e03b21 100644 --- a/.github/workflows/MainDistributionPipeline.yml +++ b/.github/workflows/MainDistributionPipeline.yml @@ -14,17 +14,17 @@ concurrency: jobs: duckdb-stable-build: name: Build extension binaries - uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@v1.5.1 + uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@v1.5-variegata with: - duckdb_version: v1.5.1 - ci_tools_version: v1.5.1 - extension_name: quack + duckdb_version: v1.5.2 + ci_tools_version: v1.5-variegata + extension_name: cozip code-quality-check: name: Code Quality Check - uses: duckdb/extension-ci-tools/.github/workflows/_extension_code_quality.yml@v1.5.1 + uses: duckdb/extension-ci-tools/.github/workflows/_extension_code_quality.yml@v1.5-variegata with: - duckdb_version: v1.5.1 - ci_tools_version: v1.5.1 - extension_name: quack + duckdb_version: v1.5.2 + ci_tools_version: v1.5-variegata + extension_name: cozip format_checks: 'format;tidy' diff --git a/.gitignore b/.gitignore index b9f264b..eef23b6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,22 @@ -build -.idea -cmake-build-debug +build/ +out/ +.vs/ +.idea/ +cmake-build-debug/ duckdb_unittest_tempdir/ -.DS_Store testext + +.DS_Store +*.user +CMakeUserPresets.json + +__pycache__/ test/python/__pycache__/ +.ipynb_checkpoints/ .Rhistory +tmp/ +notebooks/cozip_validation_artifacts/ +notebooks/ + +src/cozip_extension_old.cpp +test/sql/cozip_old.test diff --git a/CMakeLists.txt b/CMakeLists.txt index 28e6806..697159b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,12 +1,10 @@ cmake_minimum_required(VERSION 3.5) # Set extension name here -set(TARGET_NAME quack) +set(TARGET_NAME cozip) -# DuckDB's extension distribution supports vcpkg. As such, dependencies can be added in ./vcpkg.json and then -# used in cmake with find_package. Feel free to remove or replace with other dependencies. -# Note that it should also be removed from vcpkg.json to prevent needlessly installing it.. -find_package(OpenSSL REQUIRED) +# DuckDB's extension distribution supports vcpkg. cozip currently has no +# third-party dependencies. set(EXTENSION_NAME ${TARGET_NAME}_extension) set(LOADABLE_EXTENSION_NAME ${TARGET_NAME}_loadable_extension) @@ -18,15 +16,11 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON) include_directories(src/include) -set(EXTENSION_SOURCES src/quack_extension.cpp) +set(EXTENSION_SOURCES src/cozip_extension.cpp) build_static_extension(${TARGET_NAME} ${EXTENSION_SOURCES}) build_loadable_extension(${TARGET_NAME} " " ${EXTENSION_SOURCES}) -# Link OpenSSL in both the static library as the loadable extension -target_link_libraries(${EXTENSION_NAME} OpenSSL::SSL OpenSSL::Crypto) -target_link_libraries(${LOADABLE_EXTENSION_NAME} OpenSSL::SSL OpenSSL::Crypto) - install( TARGETS ${EXTENSION_NAME} EXPORT "${DUCKDB_EXPORT_SET}" diff --git a/Makefile b/Makefile index e91db43..b8e8724 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,8 @@ PROJ_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) # Configuration of extension -EXT_NAME=quack +EXT_NAME=cozip EXT_CONFIG=${PROJ_DIR}extension_config.cmake # Include the Makefile from extension-ci-tools -include extension-ci-tools/makefiles/duckdb_extension.Makefile \ No newline at end of file +include extension-ci-tools/makefiles/duckdb_extension.Makefile diff --git a/README.md b/README.md new file mode 100644 index 0000000..35378b8 --- /dev/null +++ b/README.md @@ -0,0 +1,133 @@ +# cozip DuckDB Extension + +`cozip` is a small DuckDB extension for inspecting ZIP-compatible `cozip` +archives. A `cozip` file is still a valid ZIP file, but its first entry is a +special `__cozip__` member containing a compact binary index with priority file +names, absolute payload offsets, and payload sizes. + +This extension currently focuses on local-file inspection. It validates the +core cozip 1.0 layout and exposes the priority index through DuckDB SQL. It +does not create archives or decompress payloads. + +## SQL API + +```sql +LOAD cozip; + +SELECT cozip_priority_count('test/data/flat_store_6files.cozip'); +SELECT cozip_profile('test/data/flat_store_6files.cozip'); +SELECT cozip_integrity_hash_ok('test/data/flat_store_6files.cozip'); +SELECT cozip_metadata_offset('test/data/flat_store_6files.cozip'); +SELECT cozip_index_json('test/data/flat_store_6files.cozip'); + +SELECT * +FROM cozip_index('test/data/flat_store_6files.cozip') +ORDER BY ordinal; +``` + +`cozip_index(path)` returns: + +| column | type | meaning | +| --- | --- | --- | +| `ordinal` | `UBIGINT` | priority-entry order in the `__cozip__` index | +| `name` | `VARCHAR` | priority-entry name | +| `offset_bytes` | `UBIGINT` | absolute payload offset from byte 0 | +| `size_bytes` | `UBIGINT` | payload size in bytes | + +Scalar functions: + +| function | returns | +| --- | --- | +| `cozip_priority_count(path)` | number of entries listed in the `__cozip__` priority index | +| `cozip_profile(path)` | profile byte from the cozip index header | +| `cozip_metadata_offset(path)` | absolute payload offset for `__metadata__` | +| `cozip_integrity_hash_ok(path)` | whether the stored FNV-1a integrity hash matches | +| `cozip_tail_hash_ok(path)` | compatibility alias for `cozip_integrity_hash_ok` | +| `cozip_index_json(path)` | compact JSON representation of the parsed index | + +## Format Notes + +The reader expects the current cozip core format: + +1. The first ZIP Local File Header starts at byte `0`. +2. The first entry name is exactly `__cozip__`. +3. The `__cozip__` Local File Header has a single 12-byte extra field with + header id `0xCA0C`. +4. The cozip index payload starts at byte `51`. +5. The index payload begins with `CZIP`, followed by `version`, `profile`, and + `n_entries`. +6. The integrity hash is FNV-1a 64 over the index payload and the final + `32768` bytes of the archive, counting overlapping bytes once. +7. The stored hash lives in archive bytes `43..50`. + +## Build On Windows + +Configure with CMake and the DuckDB submodule: + +```powershell +cmake -S .\duckdb -B .\build\release -G "Visual Studio 18 2026" -A x64 -T v143 ` + -DDUCKDB_EXTENSION_CONFIGS="${PWD}\extension_config.cmake" ` + -DEXTENSION_STATIC_BUILD=1 ` + -DUNITTEST_ROOT_DIRECTORY="${PWD}" ` + -DBENCHMARK_ROOT_DIRECTORY="${PWD}" ` + -DENABLE_UNITTEST_CPP_TESTS=FALSE ` + -DENABLE_EXTENSION_AUTOLOADING=0 ` + -DENABLE_EXTENSION_AUTOINSTALL=0 +``` + +If `cmake` is not on your `PATH`, use the Visual Studio bundled executable, +for example: + +```powershell +& "C:\Program Files\Microsoft Visual Studio\18\Community\Common7\IDE\CommonExtensions\Microsoft\CMake\CMake\bin\cmake.exe" --version +``` + +Build the extension and test runner: + +```powershell +cmake --build .\build\release --config Release --target cozip_loadable_extension +cmake --build .\build\release --config Release --target unittest +``` + +Run the extension tests: + +```powershell +.\build\release\test\Release\unittest.exe --test-dir . "test/sql/cozip.test" +``` + +If Windows refuses to overwrite `cozip.duckdb_extension`, close any Python, R, +or DuckDB process that previously loaded it. + +## Local Manual Test + +```powershell +.\build\release\Release\duckdb.exe -unsigned -c "LOAD './build/release/extension/cozip/cozip.duckdb_extension'; SELECT * FROM cozip_index('test/data/flat_store_6files.cozip');" +``` + +## GitHub Actions + +`.github/workflows/MainDistributionPipeline.yml` calls DuckDB's official +`extension-ci-tools` reusable pipeline. That pipeline builds and tests the +extension on Linux, Windows, and macOS using the same SQLLogicTest file under +`test/sql/` that we run locally. + +## Notebooks + +- `notebooks/cozip_architecture.ipynb` explains the binary layout, formulas, + integrity hash, and generation steps. +- `notebooks/cozip_python_testing.ipynb` loads the compiled extension from + Python and cross-checks the fixture with the pure-Python parser in + `notebooks/cozip.py`. + +## Generating A `.cozip` + +A `.cozip` is not produced by renaming an arbitrary `.zip`. It is generated as a +ZIP file with extra rules: + +1. Write `__cozip__` as the first local file entry. +2. Use ZIP method STORE for `__cozip__`. +3. Put a 12-byte extra field in the `__cozip__` local header. +4. Store a `CZIP` payload listing priority names, offsets, and sizes. +5. Write regular ZIP entries, Central Directory, and EOCD. +6. Compute FNV-1a over the index payload and the final 32768 bytes. +7. Store that hash in the `__cozip__` extra field. diff --git a/description.yml b/description.yml new file mode 100644 index 0000000..53261f5 --- /dev/null +++ b/description.yml @@ -0,0 +1,23 @@ +extension: + name: cozip + description: Read and validate cozip priority indexes from DuckDB + version: 0.1.0 + language: C++ + build: cmake + license: MIT + maintainers: + - Asterisk Labs +repo: + github: asterisk-labs/cozip_reader + ref: main +docs: + hello_world: | + LOAD cozip; + SELECT * FROM cozip_index('test/data/flat_store_6files.cozip'); + SELECT cozip_integrity_hash_ok('test/data/flat_store_6files.cozip'); + extended_description: | + cozip is a ZIP-compatible layout that places a compact binary index in the first entry + (__cozip__) so readers can locate priority files without scanning the Central Directory first. + This extension validates the current cozip core format, checks the FNV-1a integrity hash over + the index payload and archive tail, and exposes the priority index through scalar functions and + the cozip_index(path) table function. diff --git a/docs/NEXT_README.md b/docs/NEXT_README.md deleted file mode 100644 index 910dd80..0000000 --- a/docs/NEXT_README.md +++ /dev/null @@ -1,102 +0,0 @@ -# Quack - -This repository is based on https://github.com/duckdb/extension-template, check it out if you want to build and ship your own DuckDB extension. - ---- - -This extension, Quack, allow you to ... . - - -## Building -### Managing dependencies -DuckDB extensions uses VCPKG for dependency management. Enabling VCPKG is very simple: follow the [installation instructions](https://vcpkg.io/en/getting-started) or just run the following: -```shell -git clone https://github.com/Microsoft/vcpkg.git -./vcpkg/bootstrap-vcpkg.sh -export VCPKG_TOOLCHAIN_PATH=`pwd`/vcpkg/scripts/buildsystems/vcpkg.cmake -``` -Note: VCPKG is only required for extensions that want to rely on it for dependency management. If you want to develop an extension without dependencies, or want to do your own dependency management, just skip this step. Note that the example extension uses VCPKG to build with a dependency for instructive purposes, so when skipping this step the build may not work without removing the dependency. - -### Build steps -Now to build the extension, run: -```sh -make -``` -The main binaries that will be built are: -```sh -./build/release/duckdb -./build/release/test/unittest -./build/release/extension/quack/quack.duckdb_extension -``` -- `duckdb` is the binary for the duckdb shell with the extension code automatically loaded. -- `unittest` is the test runner of duckdb. Again, the extension is already linked into the binary. -- `quack.duckdb_extension` is the loadable binary as it would be distributed. - -## Running the extension -To run the extension code, simply start the shell with `./build/release/duckdb`. - -Now we can use the features from the extension directly in DuckDB. The template contains a single scalar function `quack()` that takes a string arguments and returns a string: -``` -D select quack('Jane') as result; -┌───────────────┐ -│ result │ -│ varchar │ -├───────────────┤ -│ Quack Jane 🐥 │ -└───────────────┘ -``` - -## Running the tests -Different tests can be created for DuckDB extensions. The primary way of testing DuckDB extensions should be the SQL tests in `./test/sql`. These SQL tests can be run using: -```sh -make test -``` - -### Installing the deployed binaries -To install your extension binaries from S3, you will need to do two things. Firstly, DuckDB should be launched with the -`allow_unsigned_extensions` option set to true. How to set this will depend on the client you're using. Some examples: - -CLI: -```shell -duckdb -unsigned -``` - -Python: -```python -con = duckdb.connect(':memory:', config={'allow_unsigned_extensions' : 'true'}) -``` - -NodeJS: -```js -db = new duckdb.Database(':memory:', {"allow_unsigned_extensions": "true"}); -``` - -Secondly, you will need to set the repository endpoint in DuckDB to the HTTP url of your bucket + version of the extension -you want to install. To do this run the following SQL query in DuckDB: -```sql -SET custom_extension_repository='bucket.s3.eu-west-1.amazonaws.com//latest'; -``` -Note that the `/latest` path will allow you to install the latest extension version available for your current version of -DuckDB. To specify a specific version, you can pass the version instead. - -After running these steps, you can install and load your extension using the regular INSTALL/LOAD commands in DuckDB: -```sql -INSTALL quack; -LOAD quack; -``` - -## Setting up CLion - -### Opening project -Configuring CLion with this extension requires a little work. Firstly, make sure that the DuckDB submodule is available. -Then make sure to open `./duckdb/CMakeLists.txt` (so not the top level `CMakeLists.txt` file from this repo) as a project in CLion. -Now to fix your project path go to `tools->CMake->Change Project Root`([docs](https://www.jetbrains.com/help/clion/change-project-root-directory.html)) to set the project root to the root dir of this repo. - -### Debugging -To set up debugging in CLion, there are two simple steps required. Firstly, in `CLion -> Settings / Preferences -> Build, Execution, Deploy -> CMake` you will need to add the desired builds (e.g. Debug, Release, RelDebug, etc). There's different ways to configure this, but the easiest is to leave all empty, except the `build path`, which needs to be set to `../build/{build type}`, and CMake Options to which the following flag should be added, with the path to the extension CMakeList: - -``` --DDUCKDB_EXTENSION_CONFIGS= -``` - -The second step is to configure the unittest runner as a run/debug configuration. To do this, go to `Run -> Edit Configurations` and click `+ -> Cmake Application`. The target and executable should be `unittest`. This will run all the DuckDB tests. To specify only running the extension specific tests, add `--test-dir ../../.. [sql]` to the `Program Arguments`. Note that it is recommended to use the `unittest` executable for testing/development within CLion. The actual DuckDB CLI currently does not reliably work as a run target in CLion. diff --git a/docs/README.md b/docs/README.md index a4b5f4f..14e35da 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,185 +1,42 @@ -# DuckDB Extension Template -This repository contains a template for creating a DuckDB extension. The main goal of this template is to allow users to easily develop, test and distribute their own DuckDB extension. The main branch of the template is always based on the latest stable DuckDB allowing you to try out your extension right away. +# cozip DuckDB Extension -## Getting started -First step to getting started is to create your own repo from this template by clicking `Use this template`. Then clone your new repository using -```sh -git clone --recurse-submodules https://github.com//.git -``` -Note that `--recurse-submodules` will ensure DuckDB is pulled which is required to build the extension. - -## Building -### Managing dependencies -> [!IMPORTANT] -> The example extension uses VCPKG to build with a dependency for instructive purposes, so when skipping this step the build may not work without removing the dependency. - -DuckDB extensions uses VCPKG for dependency management. Enabling VCPKG is very simple: follow the [installation instructions](https://vcpkg.io/en/getting-started) or just run the following: -```shell -cd -git clone https://github.com/Microsoft/vcpkg.git -cd vcpkg && git checkout ce613c41372b23b1f51333815feb3edd87ef8a8b -sh ./scripts/bootstrap.sh -disableMetrics -export VCPKG_TOOLCHAIN_PATH=`pwd`/vcpkg/scripts/buildsystems/vcpkg.cmake -``` - -> [!NOTE] -> VCPKG is only required for extensions that want to rely on it for dependency management. If you want to develop an extension without dependencies, or want to do your own dependency management, just skip this step. - -### Updating Submodules -DuckDB extensions use two submodules that are included in your forked extension repo when you use the `--recurse-submodules` flag. These modules are: - -| Name | Repository | Description | -|-----------------------|-------------------------------------------------|-------------| -| duckdb | https://github.com/duckdb/duckdb | This repository contains core DuckDB code required for building extensions. | -| extension-ci-tools | https://github.com/duckdb/extension-ci-tools | This repository contains reusable components for building, testing and deploying DuckDB extensions. | - - -> [!IMPORTANT] -> It is recommended that you update your submodules at least once every other major LTS release to avoid CI/CD pipeline build errors caused by remaining pinned to a stale commit of these submodules. - -To update all submodules to the latest commit hash: -```bash -git submodule update --init --recursive -``` - -To update your submodules to a specific commit hash, for example to update duckdb to the hash `8e146474d7adb960c5a2941142fe4482cc7dfc08`: -```bash -cd duckdb -git fetch --all -git checkout 8e146474d7adb960c5a2941142fe4482cc7dfc08 # or any tag/branch/commit hash -cd .. -git add duckdb -git commit -m "Pin DuckDB submodule to cc7dfc08" -git push HEAD:update-submodule-branch -``` - -### Build steps -Now to build the extension, run: -```sh -make -``` -The main binaries that will be built are: -```sh -./build/release/duckdb -./build/release/test/unittest -./build/release/extension//.duckdb_extension -``` -- `duckdb` is the binary for the duckdb shell with the extension code automatically loaded. -- `unittest` is the test runner of duckdb. Again, the extension is already linked into the binary. -- `.duckdb_extension` is the loadable binary as it would be distributed. - -### Tips for speedy builds -DuckDB extensions currently rely on DuckDB's build system to provide easy testing and distributing. This does however come at the downside of requiring the template to build DuckDB and its unittest binary every time you build your extension. To mitigate this, we highly recommend installing [ccache](https://ccache.dev/) and [ninja](https://ninja-build.org/). This will ensure you only need to build core DuckDB once and allows for rapid rebuilds. - -To build using ninja and ccache ensure both are installed and run: +This repository contains a DuckDB extension that reads the cozip core index +from ZIP-compatible archives produced by the TACO/cozip writer. -```sh -GEN=ninja make -``` - -## Running the extension -To run the extension code, simply start the shell with `./build/release/duckdb`. This shell will have the extension pre-loaded. - -Now we can use the features from the extension directly in DuckDB. The template contains a single scalar function `quack()` that takes a string arguments and returns a string: -``` -D select quack('Jane') as result; -┌───────────────┐ -│ result │ -│ varchar │ -├───────────────┤ -│ Quack Jane 🐥 │ -└───────────────┘ -``` - -## Running the tests -Different tests can be created for DuckDB extensions. The primary way of testing DuckDB extensions should be the SQL tests in `./test/sql`. These SQL tests can be run using: -```sh -make test -``` - -## Getting started with your own extension -After creating a repository from this template, the first step is to name your extension. To rename the extension, run: -```sh -# Note: This will rewrite this file! -python3 ./scripts/bootstrap-template.py -``` -Feel free to delete the script after this step. +The extension currently exposes inspection functions rather than a full TACO +table reader. It validates the byte-zero `__cozip__` entry, parses the `CZIP` +priority index, and checks the FNV-1a 64 integrity hash defined by the current +cozip 1.0 draft. -Now you're good to go! After a (re)build, you should now be able to use your duckdb extension: -``` -./build/release/duckdb -D select ('Jane') as result; -┌─────────────────────────────────────┐ -│ result │ -│ varchar │ -├─────────────────────────────────────┤ -│ Jane 🐥 │ -└─────────────────────────────────────┘ -``` +## Functions -For inspiration/examples on how to extend DuckDB in a more meaningful way, check out the [test extensions](https://github.com/duckdb/duckdb/blob/main/test/extension), -the [in-tree extensions](https://github.com/duckdb/duckdb/tree/main/extension), and the [out-of-tree extensions](https://github.com/duckdblabs). +```sql +LOAD cozip; -## Distributing your extension -To distribute your extension binaries, there are a few options. +SELECT cozip_priority_count('dataset.zip'); +SELECT cozip_profile('dataset.zip'); +SELECT cozip_integrity_hash_ok('dataset.zip'); +SELECT cozip_metadata_offset('dataset.zip'); +SELECT cozip_index_json('dataset.zip'); -### Community extensions -The recommended way of distributing extensions is through the [community extensions repository](https://github.com/duckdb/community-extensions). -This repository is designed specifically for extensions that are built using this extension template, meaning that as long as your extension can be -built using the default CI in this template, submitting it to the community extensions is a very simple process. The process works similarly to popular -package managers like homebrew and vcpkg, where a PR containing a descriptor file is submitted to the package manager repository. After the CI in the -community extensions repository completes, the extension can be installed and loaded in DuckDB with: -```SQL -INSTALL FROM community; -LOAD +SELECT * +FROM cozip_index('dataset.zip') +ORDER BY ordinal; ``` -For more information, see the [community extensions documentation](https://duckdb.org/community_extensions/documentation). -### Downloading artifacts from GitHub -The default CI in this template will automatically upload the binaries for every push to the main branch as GitHub Actions artifacts. These -can be downloaded manually and then loaded directly using: -```SQL -LOAD '/path/to/downloaded/extension.duckdb_extension'; -``` -Note that this will require starting DuckDB with the -`allow_unsigned_extensions` option set to true. How to set this will depend on the client you're using. For the CLI it is done like: -```shell -duckdb -unsigned -``` +`cozip_index(path)` returns `ordinal`, `name`, `offset_bytes`, and +`size_bytes` for each priority entry in the archive. -### Uploading to a custom repository -If for some reason distributing through community extensions is not an option, extensions can also be uploaded to a custom extension repository. -This will give some more control over where and how the extensions are distributed, but comes with the downside of requiring the `allow_unsigned_extensions` -option to be set. For examples of how to configure a manual GitHub Actions deploy pipeline, check out the extension deploy script in https://github.com/duckdb/extension-ci-tools. -Some examples of extensions that use this CI/CD workflow check out [spatial](https://github.com/duckdblabs/duckdb_spatial) or [aws](https://github.com/duckdb/duckdb_aws). +## Development Test -Extensions in custom repositories can be installed and loaded using: -```SQL -INSTALL FROM 'http://my-custom-repo' -LOAD +```powershell +cmake --build .\build\release --config MinSizeRel --target cozip_loadable_extension unittest +.\build\release\test\MinSizeRel\unittest.exe --test-dir . "test/sql/cozip.test" ``` -### Versioning of your extension -Extension binaries will only work for the specific DuckDB version they were built for. The version of DuckDB that is targeted -is set to the latest stable release for the main branch of the template so initially that is all you need. As new releases -of DuckDB are published however, the extension repository will need to be updated. The template comes with a workflow set-up -that will automatically build the binaries for all DuckDB target architectures that are available in the corresponding DuckDB -version. This workflow is found in `.github/workflows/MainDistributionPipeline.yml`. It is up to the extension developer to keep -this up to date with DuckDB. Note also that its possible to distribute binaries for multiple DuckDB versions in this workflow -by simply duplicating the jobs. - -## Setting up CLion - -### Opening project -Configuring CLion with the extension template requires a little work. Firstly, make sure that the DuckDB submodule is available. -Then make sure to open `./duckdb/CMakeLists.txt` (so not the top level `CMakeLists.txt` file from this repo) as a project in CLion. -Now to fix your project path go to `tools->CMake->Change Project Root`([docs](https://www.jetbrains.com/help/clion/change-project-root-directory.html)) to set the project root to the root dir of this repo. - -### Debugging -To set up debugging in CLion, there are two simple steps required. Firstly, in `CLion -> Settings / Preferences -> Build, Execution, Deploy -> CMake` you will need to add the desired builds (e.g. Debug, Release, RelDebug, etc). There's different ways to configure this, but the easiest is to leave all empty, except the `build path`, which needs to be set to `../build/{build type}`, and CMake Options to which the following flag should be added, with the path to the extension CMakeList: +If `cmake` is not on the active conda environment's `PATH`, use the Visual +Studio bundled executable directly: +```powershell +& "C:\Program Files\Microsoft Visual Studio\18\Community\Common7\IDE\CommonExtensions\Microsoft\CMake\CMake\bin\cmake.exe" --build .\build\release --config MinSizeRel --target cozip_loadable_extension unittest ``` --DDUCKDB_EXTENSION_CONFIGS= -``` - -The second step is to configure the unittest runner as a run/debug configuration. To do this, go to `Run -> Edit Configurations` and click `+ -> Cmake Application`. The target and executable should be `unittest`. This will run all the DuckDB tests. To specify only running the extension specific tests, add `--test-dir ../../.. [sql]` to the `Program Arguments`. Note that it is recommended to use the `unittest` executable for testing/development within CLion. The actual DuckDB CLI currently does not reliably work as a run target in CLion. diff --git a/duckdb b/duckdb index 7dbb2e6..8a58519 160000 --- a/duckdb +++ b/duckdb @@ -1 +1 @@ -Subproject commit 7dbb2e646fea939a89f10a55aa98c474cbb0c098 +Subproject commit 8a5851971fae891f292c2714d86046ee018e9737 diff --git a/extension-ci-tools b/extension-ci-tools index ef15a2a..ec20f45 160000 --- a/extension-ci-tools +++ b/extension-ci-tools @@ -1 +1 @@ -Subproject commit ef15a2a7453db5b4f85b7c668a545ae2f1193ff6 +Subproject commit ec20f45aabeb9fcfcfa044dda249597f066d4826 diff --git a/extension_config.cmake b/extension_config.cmake index ca93ea7..197fc25 100644 --- a/extension_config.cmake +++ b/extension_config.cmake @@ -1,9 +1,10 @@ # This file is included by DuckDB's build system. It specifies which extension to load # Extension from this repo -duckdb_extension_load(quack +duckdb_extension_load(cozip SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR} + LOAD_TESTS ) # Any extra extensions that should be built -# e.g.: duckdb_extension_load(json) \ No newline at end of file +# e.g.: duckdb_extension_load(json) diff --git a/scripts/bootstrap-template.py b/scripts/bootstrap-template.py deleted file mode 100755 index 30df0c9..0000000 --- a/scripts/bootstrap-template.py +++ /dev/null @@ -1,198 +0,0 @@ -#!/usr/bin/python3 - -import os -import re -import shutil -import sys -from pathlib import Path - - -def is_snake_case(s: str) -> bool: - """ - Check if the provided string is in snake_case format. - Snake case is lower case with words separated by underscores, and it can contain digits. - - Args: - s (str): String to check. - - Returns: - bool: True if the string is in snake_case, False otherwise. - """ - pattern = r"^[a-z0-9]+(_[a-z0-9]+)*$" - return bool(re.match(pattern, s)) - - -def to_camel_case(snake_str: str) -> str: - """ - Convert a snake_case string to camelCase. - - Args: - snake_str (str): String in snake_case to convert. - - Returns: - str: Converted string in camelCase. - """ - return "".join(x.capitalize() for x in snake_str.lower().split("_")) - - -def replace(file_name: str, to_find: str, to_replace: str) -> None: - """ - Replace occurrences of a string within a file, ensuring placeholders are handled. - The function replaces the `to_find` string with `to_replace`, adds a placeholder, - and skips lines with placeholders already in place. - - Args: - file_name (str): Path to the file to perform replacement in. - to_find (str): String to search for in the file. - to_replace (str): String to replace `to_find` with. - - Returns: - None - """ - with open(file_name, "r", encoding="utf8") as file: - filedata = file.readlines() - - new_filedata = [] - for line in filedata: - # Skip lines that have already been replaced by checking for placeholder - if "__REPLACEMENT_DONE__" in line: - new_filedata.append(line) - continue - - modified_line = line.replace( - to_find, - to_replace, - ) - modified_line = modified_line.replace( - to_find.capitalize(), to_camel_case(to_replace) - ) - modified_line = modified_line.replace( - to_find.upper(), - to_replace.upper(), - ) - - # Add placeholder once after all replacements - if to_find in line or to_find.capitalize() in line or to_find.upper() in line: - modified_line += "__REPLACEMENT_DONE__" - - new_filedata.append(modified_line) - - with open(file_name, "w", encoding="utf8") as file: - file.writelines(new_filedata) - - -def replace_everywhere(to_find: str, to_replace: str) -> None: - """ - Replace a string in all files in the project. - - Args: - to_find (str): String to search for in the file. - to_replace (str): String to replace `to_find` with. - - Returns: - None - """ - for path in files_to_search: - replace(path, to_find, to_replace) - replace(path, to_find.capitalize(), to_camel_case(to_replace)) - replace(path, to_find.upper(), to_replace.upper()) - - replace("./CMakeLists.txt", to_find, to_replace) - replace("./Makefile", to_find, to_replace) - replace("./Makefile", to_find.capitalize(), to_camel_case(to_replace)) - replace("./Makefile", to_find.upper(), to_replace.upper()) - replace("./README.md", to_find, to_replace) - replace("./extension_config.cmake", to_find, to_replace) - replace(".github/workflows/MainDistributionPipeline.yml", to_find, to_replace) - - -def remove_placeholder() -> None: - """ - Remove the placeholder from all files. - - Returns: - None - """ - for path in files_to_search: - replace_placeholders(path) - - replace_placeholders("./CMakeLists.txt") - replace_placeholders("./Makefile") - replace_placeholders("./Makefile") - replace_placeholders("./Makefile") - replace_placeholders("./README.md") - replace_placeholders("./extension_config.cmake") - - -def replace_placeholders(file_name: str) -> None: - """ - Remove the placeholder from a file. - - Args: - file_name (str): Path to the file to remove the placeholder from. - - Returns: - None - """ - with open(file_name, "r", encoding="utf8") as file: - filedata = file.read() - - # Remove all placeholders - filedata = filedata.replace("__REPLACEMENT_DONE__", "") - - with open(file_name, "w", encoding="utf8") as file: - file.write(filedata) - - -if __name__ == "__main__": - if len(sys.argv) != 2: - raise Exception( - "usage: python3 bootstrap-template.py " - ) - - name_extension = sys.argv[1] - - if name_extension[0].isdigit(): - raise Exception("Please dont start your extension name with a number.") - - if not is_snake_case(name_extension): - raise Exception( - "Please enter the name of your extension in valid snake_case containing only lower case letters and numbers" - ) - - shutil.copyfile("docs/NEXT_README.md", "README.md") - os.remove("docs/NEXT_README.md") - os.remove("docs/README.md") - - files_to_search = [] - files_to_search.extend(Path("./.github").rglob("./**/*.yml")) - files_to_search.extend(Path("./test").rglob("./**/*.test")) - files_to_search.extend(Path("./src").rglob("./**/*.hpp")) - files_to_search.extend(Path("./src").rglob("./**/*.cpp")) - files_to_search.extend(Path("./src").rglob("./**/*.txt")) - files_to_search.extend(Path("./src").rglob("./*.md")) - - replace_everywhere("quack", name_extension) - replace_everywhere("Quack", name_extension.capitalize()) - replace_everywhere("", name_extension) - - remove_placeholder() - - string_to_replace = name_extension - string_to_find = "quack" - - # rename files - os.rename(f"test/sql/{string_to_find}.test", f"test/sql/{string_to_replace}.test") - os.rename( - f"src/{string_to_find}_extension.cpp", f"src/{string_to_replace}_extension.cpp" - ) - os.rename( - f"src/include/{string_to_find}_extension.hpp", - f"src/include/{string_to_replace}_extension.hpp", - ) - - # remove template-specific files - os.remove(".github/workflows/ExtensionTemplate.yml") - - # finally, remove this bootstrap file - os.remove(__file__) diff --git a/src/cozip_extension.cpp b/src/cozip_extension.cpp new file mode 100644 index 0000000..e1a9f1f --- /dev/null +++ b/src/cozip_extension.cpp @@ -0,0 +1,440 @@ +#define DUCKDB_EXTENSION_MAIN + +#include "cozip_extension.hpp" + +#include "duckdb.hpp" +#include "duckdb/common/exception.hpp" +#include "duckdb/function/scalar_function.hpp" +#include "duckdb/function/table_function.hpp" + +#include +#include +#include +#include +#include + +namespace duckdb { + +static constexpr uint32_t ZIP_LOCAL_FILE_HEADER_SIGNATURE = 0x04034B50U; +static constexpr uint16_t ZIP_STORE_METHOD = 0; +static constexpr uint32_t ZIP32_SIZE_SENTINEL = 0xFFFFFFFFU; +static constexpr uint16_t ZIP_ENCRYPTED_FLAG = 1U << 0U; +static constexpr uint16_t ZIP_DATA_DESCRIPTOR_FLAG = 1U << 3U; +static constexpr uint16_t ZIP_STRONG_ENCRYPTION_FLAG = 1U << 6U; +static constexpr uint16_t ZIP_CD_ENCRYPTED_FLAG = 1U << 13U; +static constexpr uint16_t ZIP_FORBIDDEN_FLAGS = + ZIP_ENCRYPTED_FLAG | ZIP_DATA_DESCRIPTOR_FLAG | ZIP_STRONG_ENCRYPTION_FLAG | ZIP_CD_ENCRYPTED_FLAG; + +static constexpr idx_t ZIP_LOCAL_FILE_HEADER_SIZE = 30; +static constexpr idx_t COZIP_INDEX_OFFSET = 51; +static constexpr idx_t COZIP_INDEX_NAME_LEN = 9; +static constexpr idx_t COZIP_EXTRA_FIELD_SIZE = 12; +static constexpr idx_t COZIP_INTEGRITY_HASH_OFFSET = 43; +static constexpr uint16_t COZIP_EXTRA_HEADER_ID = 0xCA0C; +static constexpr uint16_t COZIP_EXTRA_DATA_SIZE = 8; +static constexpr idx_t COZIP_HASH_WINDOW_SIZE = 32768; +static constexpr idx_t COZIP_MIN_ARCHIVE_SIZE = COZIP_INDEX_OFFSET + COZIP_HASH_WINDOW_SIZE; +static constexpr idx_t COZIP_INDEX_HEADER_SIZE = 11; +static constexpr idx_t COZIP_MAX_PRIORITY_ENTRIES = 1000000; +static constexpr uint16_t COZIP_SUPPORTED_FORMAT_VERSION = 1; + +static const std::string COZIP_INDEX_NAME = "__cozip__"; + +struct CozipEntry { + std::string name; + uint64_t offset; + uint64_t size; +}; + +struct CozipIndex { + uint16_t version; + uint8_t profile; + uint32_t index_payload_size; + std::vector entries; + uint64_t integrity_hash_stored; + uint64_t integrity_hash_computed; +}; + +static void EnsureRange(idx_t pos, idx_t len, idx_t limit, const std::string &message) { + if (pos > limit || len > limit - pos) { + throw InvalidInputException(message); + } +} + +static uint16_t ReadU16(const std::vector &buf, idx_t pos) { + EnsureRange(pos, 2, buf.size(), "buffer too small for u16"); + return (uint16_t)buf[pos] | ((uint16_t)buf[pos + 1] << 8); +} + +static uint32_t ReadU32(const std::vector &buf, idx_t pos) { + EnsureRange(pos, 4, buf.size(), "buffer too small for u32"); + return (uint32_t)buf[pos] | ((uint32_t)buf[pos + 1] << 8) | ((uint32_t)buf[pos + 2] << 16) | + ((uint32_t)buf[pos + 3] << 24); +} + +static uint64_t ReadU64(const std::vector &buf, idx_t pos) { + auto lo = (uint64_t)ReadU32(buf, pos); + auto hi = (uint64_t)ReadU32(buf, pos + 4); + return lo | (hi << 32); +} + +static std::string ReadString(const std::vector &buf, idx_t pos, idx_t len) { + EnsureRange(pos, len, buf.size(), "buffer too small for string"); + return std::string(reinterpret_cast(buf.data() + pos), len); +} + +static uint64_t FNV1a64Update(uint64_t hash, const uint8_t *data, idx_t len) { + static constexpr uint64_t FNV_PRIME = 0x100000001B3ULL; + for (idx_t i = 0; i < len; i++) { + hash ^= (uint64_t)data[i]; + hash *= FNV_PRIME; + } + return hash; +} + +static std::vector ReadFileFully(const std::string &path) { + std::ifstream in(path, std::ios::binary | std::ios::ate); + if (!in) { + throw IOException("Could not open file: " + path); + } + + auto end_pos = in.tellg(); + if (end_pos < 0) { + throw IOException("Could not determine file size: " + path); + } + + std::vector data((size_t)end_pos); + in.seekg(0, std::ios::beg); + if (!data.empty() && !in.read(reinterpret_cast(data.data()), (std::streamsize)data.size())) { + throw IOException("Could not read file: " + path); + } + return data; +} + +static std::string JsonEscape(const std::string &s) { + std::ostringstream out; + for (auto c : s) { + switch (c) { + case '\\': + out << "\\\\"; + break; + case '"': + out << "\\\""; + break; + case '\n': + out << "\\n"; + break; + case '\r': + out << "\\r"; + break; + case '\t': + out << "\\t"; + break; + default: + out << c; + break; + } + } + return out.str(); +} + +static uint64_t ComputeIntegrityHash(const std::vector &buf, idx_t index_payload_size) { + static constexpr uint64_t FNV_OFFSET_BASIS = 0xCBF29CE484222325ULL; + + auto archive_size = (idx_t)buf.size(); + auto index_start = COZIP_INDEX_OFFSET; + auto index_end = COZIP_INDEX_OFFSET + index_payload_size; + auto suffix_start = archive_size - COZIP_HASH_WINDOW_SIZE; + + auto hash = FNV_OFFSET_BASIS; + if (index_end <= suffix_start) { + hash = FNV1a64Update(hash, buf.data() + index_start, index_payload_size); + hash = FNV1a64Update(hash, buf.data() + suffix_start, COZIP_HASH_WINDOW_SIZE); + } else { + hash = FNV1a64Update(hash, buf.data() + index_start, archive_size - index_start); + } + return hash; +} + +static CozipIndex ParseCozip(const std::string &path) { + auto buf = ReadFileFully(path); + if (buf.size() < COZIP_INDEX_OFFSET) { + throw InvalidInputException("Archive too small to contain a cozip index Local File Header: " + path); + } + if (buf.size() < COZIP_MIN_ARCHIVE_SIZE) { + throw InvalidInputException("Archive is smaller than the cozip minimum size of 32819 bytes: " + path); + } + if (ReadU32(buf, 0) != ZIP_LOCAL_FILE_HEADER_SIGNATURE) { + throw InvalidInputException("Byte 0 is not a ZIP local file header: " + path); + } + + auto flags = ReadU16(buf, 6); + auto method = ReadU16(buf, 8); + auto compressed_size = ReadU32(buf, 18); + auto uncompressed_size = ReadU32(buf, 22); + auto name_len = ReadU16(buf, 26); + auto extra_len = ReadU16(buf, 28); + + if ((flags & ZIP_FORBIDDEN_FLAGS) != 0) { + throw InvalidInputException("cozip index entry has forbidden ZIP general-purpose flags: " + path); + } + if (method != ZIP_STORE_METHOD) { + throw InvalidInputException("cozip index entry must use ZIP STORE compression method: " + path); + } + if (compressed_size == 0 || compressed_size == ZIP32_SIZE_SENTINEL || compressed_size != uncompressed_size) { + throw InvalidInputException("cozip index entry has invalid ZIP32 size fields: " + path); + } + if (name_len != COZIP_INDEX_NAME_LEN) { + throw InvalidInputException("cozip index entry filename length must be 9 bytes: " + path); + } + if (extra_len != COZIP_EXTRA_FIELD_SIZE) { + throw InvalidInputException("cozip index entry must have a 12-byte local extra field: " + path); + } + if (ReadString(buf, ZIP_LOCAL_FILE_HEADER_SIZE, COZIP_INDEX_NAME_LEN) != COZIP_INDEX_NAME) { + throw InvalidInputException("First ZIP entry is not __cozip__: " + path); + } + if (ReadU16(buf, 39) != COZIP_EXTRA_HEADER_ID || ReadU16(buf, 41) != COZIP_EXTRA_DATA_SIZE) { + throw InvalidInputException("cozip index entry is missing the 0xCA0C integrity extra field: " + path); + } + + auto index_payload_size = (idx_t)compressed_size; + auto payload_offset = COZIP_INDEX_OFFSET; + auto payload_end = payload_offset + index_payload_size; + EnsureRange(payload_offset, index_payload_size, buf.size(), "cozip index payload exceeds archive size: " + path); + EnsureRange(payload_offset, COZIP_INDEX_HEADER_SIZE, payload_end, "truncated cozip index header: " + path); + + auto pos = payload_offset; + if (ReadString(buf, pos, 4) != "CZIP") { + throw InvalidInputException("Invalid cozip magic: " + path); + } + + CozipIndex index; + index.version = ReadU16(buf, pos + 4); + index.profile = buf[pos + 6]; + index.index_payload_size = compressed_size; + auto n_entries = (idx_t)ReadU32(buf, pos + 7); + + if (index.version > COZIP_SUPPORTED_FORMAT_VERSION) { + throw InvalidInputException("Unsupported cozip format version: " + std::to_string(index.version)); + } + if (n_entries > COZIP_MAX_PRIORITY_ENTRIES) { + throw InvalidInputException("cozip index has too many priority entries: " + path); + } + pos += COZIP_INDEX_HEADER_SIZE; + + EnsureRange(pos, 2 * n_entries, payload_end, "truncated cozip name-length section: " + path); + std::vector name_lengths; + name_lengths.reserve(n_entries); + for (idx_t i = 0; i < n_entries; i++) { + name_lengths.push_back(ReadU16(buf, pos + 2 * i)); + } + pos += 2 * n_entries; + + std::vector names; + names.reserve(n_entries); + std::unordered_set seen_names; + for (idx_t i = 0; i < n_entries; i++) { + auto len = (idx_t)name_lengths[i]; + if (len == 0) { + throw InvalidInputException("cozip priority entry name cannot be empty: " + path); + } + EnsureRange(pos, len, payload_end, "truncated cozip names section: " + path); + auto name = ReadString(buf, pos, len); + if (name == COZIP_INDEX_NAME) { + throw InvalidInputException("cozip index must not list __cozip__ as a priority entry: " + path); + } + if (!seen_names.insert(name).second) { + throw InvalidInputException("duplicate cozip priority entry name: " + name); + } + names.push_back(std::move(name)); + pos += len; + } + + EnsureRange(pos, 8 * n_entries, payload_end, "truncated cozip offsets section: " + path); + std::vector offsets; + offsets.reserve(n_entries); + for (idx_t i = 0; i < n_entries; i++) { + offsets.push_back(ReadU64(buf, pos + 8 * i)); + } + pos += 8 * n_entries; + + EnsureRange(pos, 8 * n_entries, payload_end, "truncated cozip sizes section: " + path); + std::vector sizes; + sizes.reserve(n_entries); + for (idx_t i = 0; i < n_entries; i++) { + sizes.push_back(ReadU64(buf, pos + 8 * i)); + } + pos += 8 * n_entries; + if (pos != payload_end) { + throw InvalidInputException("cozip index payload has trailing bytes: " + path); + } + + index.entries.reserve(n_entries); + for (idx_t i = 0; i < n_entries; i++) { + if (sizes[i] == 0) { + throw InvalidInputException("cozip priority entry has zero payload size: " + names[i]); + } + if (offsets[i] > buf.size() || sizes[i] > buf.size() - offsets[i]) { + throw InvalidInputException("cozip priority entry exceeds archive size: " + names[i]); + } + index.entries.push_back(CozipEntry {names[i], offsets[i], sizes[i]}); + } + + index.integrity_hash_stored = ReadU64(buf, COZIP_INTEGRITY_HASH_OFFSET); + index.integrity_hash_computed = ComputeIntegrityHash(buf, index_payload_size); + + return index; +} + +inline void CozipPriorityCountFun(DataChunk &args, ExpressionState &state, Vector &result) { + auto &path_vector = args.data[0]; + UnaryExecutor::Execute(path_vector, result, args.size(), [&](string_t path) { + auto idx = ParseCozip(path.GetString()); + return (uint64_t)idx.entries.size(); + }); +} + +inline void CozipProfileFun(DataChunk &args, ExpressionState &state, Vector &result) { + auto &path_vector = args.data[0]; + UnaryExecutor::Execute(path_vector, result, args.size(), [&](string_t path) { + auto idx = ParseCozip(path.GetString()); + return (uint64_t)idx.profile; + }); +} + +inline void CozipMetadataOffsetFun(DataChunk &args, ExpressionState &state, Vector &result) { + auto &path_vector = args.data[0]; + UnaryExecutor::Execute(path_vector, result, args.size(), [&](string_t path) { + auto idx = ParseCozip(path.GetString()); + for (auto &entry : idx.entries) { + if (entry.name == "__metadata__") { + return entry.offset; + } + } + throw InvalidInputException("Priority entry '__metadata__' not found in archive: " + path.GetString()); + }); +} + +inline void CozipIntegrityHashOkFun(DataChunk &args, ExpressionState &state, Vector &result) { + auto &path_vector = args.data[0]; + UnaryExecutor::Execute(path_vector, result, args.size(), [&](string_t path) { + auto idx = ParseCozip(path.GetString()); + return idx.integrity_hash_stored == idx.integrity_hash_computed; + }); +} + +inline void CozipIndexJsonFun(DataChunk &args, ExpressionState &state, Vector &result) { + auto &path_vector = args.data[0]; + UnaryExecutor::Execute(path_vector, result, args.size(), [&](string_t path) { + auto idx = ParseCozip(path.GetString()); + + std::ostringstream out; + out << "{\"version\":" << idx.version << ",\"profile\":" << (uint64_t)idx.profile + << ",\"index_payload_size\":" << idx.index_payload_size << ",\"integrity_hash_ok\":" + << (idx.integrity_hash_stored == idx.integrity_hash_computed ? "true" : "false") << ",\"entries\":["; + for (idx_t i = 0; i < idx.entries.size(); i++) { + if (i > 0) { + out << ","; + } + out << "{\"name\":\"" << JsonEscape(idx.entries[i].name) << "\",\"offset\":" << idx.entries[i].offset + << ",\"size\":" << idx.entries[i].size << "}"; + } + out << "]}"; + + return StringVector::AddString(result, out.str()); + }); +} + +struct CozipIndexBindData : public TableFunctionData { + explicit CozipIndexBindData(CozipIndex index) : index(std::move(index)) { + } + + CozipIndex index; +}; + +struct CozipIndexGlobalState : public GlobalTableFunctionState { + idx_t offset = 0; +}; + +static duckdb::unique_ptr CozipIndexBind(ClientContext &context, TableFunctionBindInput &input, + vector &return_types, vector &names) { + if (input.inputs[0].IsNull()) { + throw BinderException("cozip_index path parameter cannot be NULL"); + } + + names.emplace_back("ordinal"); + return_types.emplace_back(LogicalType::UBIGINT); + names.emplace_back("name"); + return_types.emplace_back(LogicalType::VARCHAR); + names.emplace_back("offset_bytes"); + return_types.emplace_back(LogicalType::UBIGINT); + names.emplace_back("size_bytes"); + return_types.emplace_back(LogicalType::UBIGINT); + + auto path = StringValue::Get(input.inputs[0]); + return make_uniq(ParseCozip(path)); +} + +static duckdb::unique_ptr CozipIndexInit(ClientContext &context, + TableFunctionInitInput &input) { + return make_uniq(); +} + +static void CozipIndexFunction(ClientContext &context, TableFunctionInput &data_p, DataChunk &output) { + auto &bind_data = data_p.bind_data->Cast(); + auto &state = (CozipIndexGlobalState &)*data_p.global_state; + + idx_t count = 0; + while (state.offset < bind_data.index.entries.size() && count < STANDARD_VECTOR_SIZE) { + auto &entry = bind_data.index.entries[state.offset]; + output.SetValue(0, count, Value::UBIGINT(state.offset)); + output.SetValue(1, count, Value(entry.name)); + output.SetValue(2, count, Value::UBIGINT(entry.offset)); + output.SetValue(3, count, Value::UBIGINT(entry.size)); + state.offset++; + count++; + } + output.SetCardinality(count); +} + +static void LoadInternal(ExtensionLoader &loader) { + loader.RegisterFunction( + ScalarFunction("cozip_priority_count", {LogicalType::VARCHAR}, LogicalType::UBIGINT, CozipPriorityCountFun)); + loader.RegisterFunction( + ScalarFunction("cozip_profile", {LogicalType::VARCHAR}, LogicalType::UBIGINT, CozipProfileFun)); + loader.RegisterFunction( + ScalarFunction("cozip_metadata_offset", {LogicalType::VARCHAR}, LogicalType::UBIGINT, CozipMetadataOffsetFun)); + loader.RegisterFunction(ScalarFunction("cozip_integrity_hash_ok", {LogicalType::VARCHAR}, LogicalType::BOOLEAN, + CozipIntegrityHashOkFun)); + loader.RegisterFunction( + ScalarFunction("cozip_tail_hash_ok", {LogicalType::VARCHAR}, LogicalType::BOOLEAN, CozipIntegrityHashOkFun)); + loader.RegisterFunction( + ScalarFunction("cozip_index_json", {LogicalType::VARCHAR}, LogicalType::VARCHAR, CozipIndexJsonFun)); + loader.RegisterFunction( + TableFunction("cozip_index", {LogicalType::VARCHAR}, CozipIndexFunction, CozipIndexBind, CozipIndexInit)); +} + +void CozipExtension::Load(ExtensionLoader &loader) { + LoadInternal(loader); +} + +std::string CozipExtension::Name() { + return "cozip"; +} + +std::string CozipExtension::Version() const { +#ifdef EXT_VERSION_COZIP + return EXT_VERSION_COZIP; +#else + return ""; +#endif +} + +} // namespace duckdb + +extern "C" { + +DUCKDB_CPP_EXTENSION_ENTRY(cozip, loader) { + duckdb::LoadInternal(loader); +} +} diff --git a/src/include/quack_extension.hpp b/src/include/cozip_extension.hpp similarity index 64% rename from src/include/quack_extension.hpp rename to src/include/cozip_extension.hpp index c3c0ab6..fe99d21 100644 --- a/src/include/quack_extension.hpp +++ b/src/include/cozip_extension.hpp @@ -4,9 +4,9 @@ namespace duckdb { -class QuackExtension : public Extension { +class CozipExtension : public Extension { public: - void Load(ExtensionLoader &db) override; + void Load(ExtensionLoader &loader) override; std::string Name() override; std::string Version() const override; }; diff --git a/src/quack_extension.cpp b/src/quack_extension.cpp deleted file mode 100644 index 25c5d69..0000000 --- a/src/quack_extension.cpp +++ /dev/null @@ -1,62 +0,0 @@ -#define DUCKDB_EXTENSION_MAIN - -#include "quack_extension.hpp" -#include "duckdb.hpp" -#include "duckdb/common/exception.hpp" -#include "duckdb/function/scalar_function.hpp" -#include - -// OpenSSL linked through vcpkg -#include - -namespace duckdb { - -inline void QuackScalarFun(DataChunk &args, ExpressionState &state, Vector &result) { - auto &name_vector = args.data[0]; - UnaryExecutor::Execute(name_vector, result, args.size(), [&](string_t name) { - return StringVector::AddString(result, "Quack " + name.GetString() + " 🐥"); - }); -} - -inline void QuackOpenSSLVersionScalarFun(DataChunk &args, ExpressionState &state, Vector &result) { - auto &name_vector = args.data[0]; - UnaryExecutor::Execute(name_vector, result, args.size(), [&](string_t name) { - return StringVector::AddString(result, "Quack " + name.GetString() + ", my linked OpenSSL version is " + - OPENSSL_VERSION_TEXT); - }); -} - -static void LoadInternal(ExtensionLoader &loader) { - // Register a scalar function - auto quack_scalar_function = ScalarFunction("quack", {LogicalType::VARCHAR}, LogicalType::VARCHAR, QuackScalarFun); - loader.RegisterFunction(quack_scalar_function); - - // Register another scalar function - auto quack_openssl_version_scalar_function = ScalarFunction("quack_openssl_version", {LogicalType::VARCHAR}, - LogicalType::VARCHAR, QuackOpenSSLVersionScalarFun); - loader.RegisterFunction(quack_openssl_version_scalar_function); -} - -void QuackExtension::Load(ExtensionLoader &loader) { - LoadInternal(loader); -} -std::string QuackExtension::Name() { - return "quack"; -} - -std::string QuackExtension::Version() const { -#ifdef EXT_VERSION_QUACK - return EXT_VERSION_QUACK; -#else - return ""; -#endif -} - -} // namespace duckdb - -extern "C" { - -DUCKDB_CPP_EXTENSION_ENTRY(quack, loader) { - duckdb::LoadInternal(loader); -} -} diff --git a/test/README.md b/test/README.md index fb5e514..ba00303 100644 --- a/test/README.md +++ b/test/README.md @@ -1,11 +1,12 @@ -# Testing this extension -This directory contains all the tests for this extension. The `sql` directory holds tests that are written as [SQLLogicTests](https://duckdb.org/dev/sqllogictest/intro.html). DuckDB aims to have most its tests in this format as SQL statements, so for the quack extension, this should probably be the goal too. +# Testing cozip -The root makefile contains targets to build and run all of these tests. To run the SQLLogicTests: -```bash -make test +The `sql` directory contains SQLLogicTests for the DuckDB extension. + +On Windows, after configuring CMake, run: + +```powershell +.\build\release\test\MinSizeRel\unittest.exe --test-dir . "test/sql/cozip.test" ``` -or -```bash -make test_debug -``` \ No newline at end of file + +The current fixture in `test/data/flat_store_6files.cozip` follows the +cozip 1.0 FLAT profile produced by the writer in the sibling `taco` project. diff --git a/test/data/flat_store_6files.cozip b/test/data/flat_store_6files.cozip new file mode 100644 index 0000000..050a27c Binary files /dev/null and b/test/data/flat_store_6files.cozip differ diff --git a/test/sql/cozip.test b/test/sql/cozip.test new file mode 100644 index 0000000..0b545cc --- /dev/null +++ b/test/sql/cozip.test @@ -0,0 +1,45 @@ +# name: test/sql/cozip.test +# description: test cozip reader extension +# group: [sql] + +statement error +SELECT cozip_priority_count('test/data/flat_store_6files.cozip'); +---- +Catalog Error: Scalar Function with name cozip_priority_count does not exist! + +statement error +SELECT * FROM cozip_index('test/data/flat_store_6files.cozip'); +---- +Catalog Error: Table Function with name cozip_index does not exist! + +require cozip + +query III +SELECT + cozip_priority_count('test/data/flat_store_6files.cozip'), + cozip_profile('test/data/flat_store_6files.cozip'), + cozip_metadata_offset('test/data/flat_store_6files.cozip'); +---- +1 1 246212 + +query II +SELECT + cozip_integrity_hash_ok('test/data/flat_store_6files.cozip')::INTEGER, + cozip_tail_hash_ok('test/data/flat_store_6files.cozip')::INTEGER; +---- +1 1 + +query ITII +SELECT ordinal, name, offset_bytes, size_bytes +FROM cozip_index('test/data/flat_store_6files.cozip') +ORDER BY ordinal; +---- +0 __metadata__ 246212 3201 + +query I +SELECT ( + cozip_index_json('test/data/flat_store_6files.cozip') = + '{"version":1,"profile":1,"index_payload_size":41,"integrity_hash_ok":true,"entries":[{"name":"__metadata__","offset":246212,"size":3201}]}' +)::INTEGER; +---- +1 diff --git a/test/sql/quack.test b/test/sql/quack.test deleted file mode 100644 index 6416c65..0000000 --- a/test/sql/quack.test +++ /dev/null @@ -1,23 +0,0 @@ -# name: test/sql/quack.test -# description: test quack extension -# group: [sql] - -# Before we load the extension, this will fail -statement error -SELECT quack('Sam'); ----- -Catalog Error: Scalar Function with name quack does not exist! - -# Require statement will ensure this test is run with this extension loaded -require quack - -# Confirm the extension works -query I -SELECT quack('Sam'); ----- -Quack Sam 🐥 - -query I -SELECT quack_openssl_version('Michael') ILIKE 'Quack Michael, my linked OpenSSL version is OpenSSL%'; ----- -true diff --git a/vcpkg.json b/vcpkg.json index 115e19c..1e41e0b 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -1,13 +1,11 @@ { - "dependencies": [ - "openssl" - ], - "vcpkg-configuration": { - "overlay-ports": [ - "./extension-ci-tools/vcpkg_ports" - ], - "overlay-triplets": [ - "./extension-ci-tools/toolchains" - ] - } + "dependencies": [], + "vcpkg-configuration": { + "overlay-ports": [ + "./extension-ci-tools/vcpkg_ports" + ], + "overlay-triplets": [ + "./extension-ci-tools/toolchains" + ] + } }