Add ruby bindings

.github/workflows/build_ruby_gem.yml

@@ -0,0 +1,107 @@
+name: Build ruby gem
+on: workflow_dispatch
+env:
+  CACHE_VERSION: "v0"
+  CARGO_CACHE_CLEAN: "true"
+  RUBY_VERSIONS: "3.2"
+jobs:
+  release:
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - platform: x86_64-linux
+            target: x86_64-unknown-linux-gnu
+          - platform: x86_64-linux-musl
+            target: x86_64-unknown-linux-musl
+          - platform: aarch64-linux
+            target: aarch64-unknown-linux-gnu
+          - platform: x86_64-darwin
+            target: x86_64-apple-darwin
+            # Rust uses external command to strip symbols and debuginfo on Mac
+            # Do not do for arm64 since it interferes with code signing
+            # and codesign binary is not present to re-sign
+            setup: sudo ln -s /opt/osxcross/target/bin/x86_64-apple-darwin-strip /usr/local/bin/strip
+          - platform: arm64-darwin
+            target: aarch64-apple-darwin
+          # - platform: x64-mingw-ucrt
+          #   target: x86_64-pc-windows-gnu
+          # - platform: x64-mingw32
+          #   target: x86_64-pc-windows-gnu
+    runs-on: ubuntu-latest
+    name: ${{ matrix.platform }}
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: 3.2
+      - name: generate rank
+        run: bundle install && bundle exec rake rank
+        working-directory: "ruby"
+
+      # Didn't use https://github.com/oxidize-rb/actions/tree/main/cross-gem due to weird directory structure, but this code is adapted from there.
+      - name: Configure environment
+        run: |
+          : Configure environment
+          echo "RB_SYS_DOCK_UID=$(id -u)" >> $GITHUB_ENV
+          echo "RB_SYS_DOCK_GID=$(id -g)" >> $GITHUB_ENV
+          rb_sys_dock_cache_dir="$HOME/.cache/rb-sys-dock"
+          mkdir -p "$rb_sys_dock_cache_dir"
+          echo "RB_SYS_DOCK_CACHE_DIR=$rb_sys_dock_cache_dir" >> $GITHUB_ENV
+      - name: Setup caching
+        uses: actions/cache@v3
+        with:
+          path: |
+            ${{ env.RB_SYS_DOCK_CACHE_DIR }}
+            ${{ github.workspace }}/ruby/tmp/rb-sys-dock/${{ matrix.platform }}/target
+          key: rb-sys-dock-${{ env.CACHE_VERSION }}-${{ matrix.platform }}-${{ hashFiles('**/Gemfile.lock', '**/Cargo.lock') }}
+          restore-keys: |
+            rb-sys-dock-${{ env.CACHE_VERSION }}-${{ matrix.platform }}-
+      - name: Install cargo-cache
+        uses: oxidize-rb/actions/cargo-binstall@v1
+        id: install-cargo-cache
+        if: env.CARGO_CACHE_CLEAN == 'true'
+        with:
+          crate: cargo-cache
+          version: 0.8.3
+          strategies: quick-install
+
+      - name: Clean the cargo cache
+        if: env.CARGO_CACHE_CLEAN == 'true'
+        uses: oxidize-rb/actions/post-run@v1
+        with:
+          run: cargo-cache --autoclean
+          cwd: ${{ github.workspace }}
+
+      #- name: Start SSH session
+      #  uses: luchihoratiu/debug-via-ssh@main
+      #  with:
+      #    NGROK_AUTH_TOKEN: ${{ secrets.NGROK_AUTH_TOKEN }}
+      #    SSH_PASS: ${{ secrets.SSH_PASS }}
+
+      - name: Build gem
+        env:
+          INPUT_RUBY_VERSIONS: "${{ env.RUBY_VERSIONS }}"
+          INPUT_PLATFORM: "${{ matrix.platform }}"
+        run: |
+          : Compile gem
+          set -x
+          args=()
+          args+=("--platform")
+          args+=("$INPUT_PLATFORM")
+          if [ "$INPUT_RUBY_VERSIONS" != "default" ]; then
+            args+=("--ruby-versions")
+            args+=("$INPUT_RUBY_VERSIONS")
+          fi
+          BUNDLE_GEMFILE=ruby/Gemfile bundle exec rb-sys-dock "${args[@]}" --build -- "cd ruby && bundle install && export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc && env | grep CARGO"
+
+      - name: Set outputs
+        id: set-outputs
+        run: |
+          : Set output
+          echo "gem-path=$(find ${{ github.workspace }}/ruby/pkg -name '*-${{ matrix.platform }}.gem')" >> $GITHUB_OUTPUT
+
+      - uses: actions/upload-artifact@v3
+        with:
+          name: cross-gem
+          path: ${{ steps.set-outputs.outputs.gem-path }}

.gitignore

@@ -40,3 +40,7 @@ htmlcov
 
 Cargo.lock
 target/
+
+# WASM
+ranks/
+node_modules

Cargo.toml

@@ -1,21 +1,17 @@
-[package]
-name = "tiktoken"
-version = "0.3.0"
-edition = "2021"
-rust-version = "1.57.0"
+[workspace]
 
-[lib]
-name = "_tiktoken"
-crate-type = ["cdylib"]
-
-[dependencies]
-pyo3 = { version = "0.17.3", features = ["extension-module"] }
-
-# tiktoken dependencies
-fancy-regex = "0.10.0"
-regex = "1.7.0"
-rustc-hash = "1.1.0"
-bstr = "1.0.1"
+members = [
+    "core",
+    "jni",
+    "js",
+    "python",
+    "ruby",
+]
 
 [profile.release]
 incremental = true
+opt-level = 's'     # Optimize for size
+lto = true          # Enable link-time optimization
+codegen-units = 1   # Reduce number of codegen units to increase optimizations
+panic = 'abort'     # Abort on panic
+strip = true        # Strip symbols from binary*

MANIFEST.in

@@ -5,4 +5,9 @@ include Makefile
 global-include py.typed
 recursive-include scripts *.py
 recursive-include tests *.py
-recursive-include src *.rs
+recursive-include core *.rs *.toml
+recursive-include python *.rs *.toml
+recursive-exclude jni *
+recursive-exclude java *
+recursive-exclude js *
+include tiktoken *.json

README.md

@@ -78,7 +78,7 @@ Layout your project like this, making sure to omit the `tiktoken_ext/__init__.py
 ```
 my_tiktoken_extension
 ├── tiktoken_ext
-│   └── my_encodings.py
+│   └── my_encodings.py
 └── setup.py
 ```
 
@@ -101,4 +101,3 @@ setup(
 
 Then simply `pip install ./my_tiktoken_extension` and you should be able to use your
 custom encodings! Make sure **not** to use an editable install.
-

core/Cargo.toml

@@ -0,0 +1,26 @@
+[package]
+name = "tiktoken_core"
+version = "0.3.0"
+edition = "2021"
+rust-version = "1.57.0"
+
+[lib]
+name = "_tiktoken_core"
+crate-type = ["lib"]
+
+[dependencies]
+# tiktoken dependencies
+fancy-regex = "0.10.0"
+regex = "1.7.0"
+rustc-hash = "1.1.0"
+bstr = "1.0.1"
+reqwest = { version = "0.11.14", features = ["rustls-tls", "blocking"], default-features = false }
+sha1 = "0.10.5"
+json = "0.12.4"
+base64 = "0.21.0"
+lazy_static = "1.4.0"
+
+[features]
+default = []
+lazyload = []
+multithreading = []