Fit tokenizer

Fit tokenizer

curl --request POST \
  --url https://api.jeantechnologies.com/v1/tokenizers \
  --header 'Authorization: Bearer <token>' \
  --header 'Content-Type: application/json' \
  --data '
{
  "name": "catalog-v1",
  "catalog": {
    "uri": "s3://acme/catalog.jsonl",
    "format": "jsonl"
  },
  "modalities": [
    "text",
    "image"
  ],
  "codebook": {
    "levels": 4,
    "resolution": "multi",
    "base_size": 2048
  },
  "collaborative_signal": {
    "mode": "co_occurrence_contrastive",
    "interactions": {
      "uri": "s3://acme/sequences.jsonl",
      "format": "jsonl"
    },
    "weight": 0.3
  },
  "progressive_masking": true
}
'

import requests

url = "https://api.jeantechnologies.com/v1/tokenizers"

payload = {
    "name": "catalog-v1",
    "catalog": {
        "uri": "s3://acme/catalog.jsonl",
        "format": "jsonl"
    },
    "modalities": ["text", "image"],
    "codebook": {
        "levels": 4,
        "resolution": "multi",
        "base_size": 2048
    },
    "collaborative_signal": {
        "mode": "co_occurrence_contrastive",
        "interactions": {
            "uri": "s3://acme/sequences.jsonl",
            "format": "jsonl"
        },
        "weight": 0.3
    },
    "progressive_masking": True
}
headers = {
    "Authorization": "Bearer <token>",
    "Content-Type": "application/json"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
  body: JSON.stringify({
    name: 'catalog-v1',
    catalog: {uri: 's3://acme/catalog.jsonl', format: 'jsonl'},
    modalities: ['text', 'image'],
    codebook: {levels: 4, resolution: 'multi', base_size: 2048},
    collaborative_signal: {
      mode: 'co_occurrence_contrastive',
      interactions: {uri: 's3://acme/sequences.jsonl', format: 'jsonl'},
      weight: 0.3
    },
    progressive_masking: true
  })
};

fetch('https://api.jeantechnologies.com/v1/tokenizers', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://api.jeantechnologies.com/v1/tokenizers",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'name' => 'catalog-v1',
    'catalog' => [
        'uri' => 's3://acme/catalog.jsonl',
        'format' => 'jsonl'
    ],
    'modalities' => [
        'text',
        'image'
    ],
    'codebook' => [
        'levels' => 4,
        'resolution' => 'multi',
        'base_size' => 2048
    ],
    'collaborative_signal' => [
        'mode' => 'co_occurrence_contrastive',
        'interactions' => [
                'uri' => 's3://acme/sequences.jsonl',
                'format' => 'jsonl'
        ],
        'weight' => 0.3
    ],
    'progressive_masking' => true
  ]),
  CURLOPT_HTTPHEADER => [
    "Authorization: Bearer <token>",
    "Content-Type: application/json"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://api.jeantechnologies.com/v1/tokenizers"

	payload := strings.NewReader("{\n  \"name\": \"catalog-v1\",\n  \"catalog\": {\n    \"uri\": \"s3://acme/catalog.jsonl\",\n    \"format\": \"jsonl\"\n  },\n  \"modalities\": [\n    \"text\",\n    \"image\"\n  ],\n  \"codebook\": {\n    \"levels\": 4,\n    \"resolution\": \"multi\",\n    \"base_size\": 2048\n  },\n  \"collaborative_signal\": {\n    \"mode\": \"co_occurrence_contrastive\",\n    \"interactions\": {\n      \"uri\": \"s3://acme/sequences.jsonl\",\n      \"format\": \"jsonl\"\n    },\n    \"weight\": 0.3\n  },\n  \"progressive_masking\": true\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("Authorization", "Bearer <token>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://api.jeantechnologies.com/v1/tokenizers")
  .header("Authorization", "Bearer <token>")
  .header("Content-Type", "application/json")
  .body("{\n  \"name\": \"catalog-v1\",\n  \"catalog\": {\n    \"uri\": \"s3://acme/catalog.jsonl\",\n    \"format\": \"jsonl\"\n  },\n  \"modalities\": [\n    \"text\",\n    \"image\"\n  ],\n  \"codebook\": {\n    \"levels\": 4,\n    \"resolution\": \"multi\",\n    \"base_size\": 2048\n  },\n  \"collaborative_signal\": {\n    \"mode\": \"co_occurrence_contrastive\",\n    \"interactions\": {\n      \"uri\": \"s3://acme/sequences.jsonl\",\n      \"format\": \"jsonl\"\n    },\n    \"weight\": 0.3\n  },\n  \"progressive_masking\": true\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://api.jeantechnologies.com/v1/tokenizers")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n  \"name\": \"catalog-v1\",\n  \"catalog\": {\n    \"uri\": \"s3://acme/catalog.jsonl\",\n    \"format\": \"jsonl\"\n  },\n  \"modalities\": [\n    \"text\",\n    \"image\"\n  ],\n  \"codebook\": {\n    \"levels\": 4,\n    \"resolution\": \"multi\",\n    \"base_size\": 2048\n  },\n  \"collaborative_signal\": {\n    \"mode\": \"co_occurrence_contrastive\",\n    \"interactions\": {\n      \"uri\": \"s3://acme/sequences.jsonl\",\n      \"format\": \"jsonl\"\n    },\n    \"weight\": 0.3\n  },\n  \"progressive_masking\": true\n}"

response = http.request(request)
puts response.read_body

{
  "tokenizer_id": "tok_9k2m",
  "name": "catalog-v1",
  "status": "queued",
  "codebook": {
    "levels": 4,
    "resolution": "multi",
    "base_size": 2048
  },
  "created_at": "2026-08-31T18:04:11Z",
  "ready_at": null
}

{
  "error": {
    "code": "invalid_request",
    "message": "Field 'domain' is required.",
    "details": {}
  }
}

{
  "error": {
    "code": "invalid_request",
    "message": "Field 'domain' is required.",
    "details": {}
  }
}

POST

tokenizers

Fit tokenizer

curl --request POST \
  --url https://api.jeantechnologies.com/v1/tokenizers \
  --header 'Authorization: Bearer <token>' \
  --header 'Content-Type: application/json' \
  --data '
{
  "name": "catalog-v1",
  "catalog": {
    "uri": "s3://acme/catalog.jsonl",
    "format": "jsonl"
  },
  "modalities": [
    "text",
    "image"
  ],
  "codebook": {
    "levels": 4,
    "resolution": "multi",
    "base_size": 2048
  },
  "collaborative_signal": {
    "mode": "co_occurrence_contrastive",
    "interactions": {
      "uri": "s3://acme/sequences.jsonl",
      "format": "jsonl"
    },
    "weight": 0.3
  },
  "progressive_masking": true
}
'

import requests

url = "https://api.jeantechnologies.com/v1/tokenizers"

payload = {
    "name": "catalog-v1",
    "catalog": {
        "uri": "s3://acme/catalog.jsonl",
        "format": "jsonl"
    },
    "modalities": ["text", "image"],
    "codebook": {
        "levels": 4,
        "resolution": "multi",
        "base_size": 2048
    },
    "collaborative_signal": {
        "mode": "co_occurrence_contrastive",
        "interactions": {
            "uri": "s3://acme/sequences.jsonl",
            "format": "jsonl"
        },
        "weight": 0.3
    },
    "progressive_masking": True
}
headers = {
    "Authorization": "Bearer <token>",
    "Content-Type": "application/json"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
  body: JSON.stringify({
    name: 'catalog-v1',
    catalog: {uri: 's3://acme/catalog.jsonl', format: 'jsonl'},
    modalities: ['text', 'image'],
    codebook: {levels: 4, resolution: 'multi', base_size: 2048},
    collaborative_signal: {
      mode: 'co_occurrence_contrastive',
      interactions: {uri: 's3://acme/sequences.jsonl', format: 'jsonl'},
      weight: 0.3
    },
    progressive_masking: true
  })
};

fetch('https://api.jeantechnologies.com/v1/tokenizers', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://api.jeantechnologies.com/v1/tokenizers",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'name' => 'catalog-v1',
    'catalog' => [
        'uri' => 's3://acme/catalog.jsonl',
        'format' => 'jsonl'
    ],
    'modalities' => [
        'text',
        'image'
    ],
    'codebook' => [
        'levels' => 4,
        'resolution' => 'multi',
        'base_size' => 2048
    ],
    'collaborative_signal' => [
        'mode' => 'co_occurrence_contrastive',
        'interactions' => [
                'uri' => 's3://acme/sequences.jsonl',
                'format' => 'jsonl'
        ],
        'weight' => 0.3
    ],
    'progressive_masking' => true
  ]),
  CURLOPT_HTTPHEADER => [
    "Authorization: Bearer <token>",
    "Content-Type: application/json"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://api.jeantechnologies.com/v1/tokenizers"

	payload := strings.NewReader("{\n  \"name\": \"catalog-v1\",\n  \"catalog\": {\n    \"uri\": \"s3://acme/catalog.jsonl\",\n    \"format\": \"jsonl\"\n  },\n  \"modalities\": [\n    \"text\",\n    \"image\"\n  ],\n  \"codebook\": {\n    \"levels\": 4,\n    \"resolution\": \"multi\",\n    \"base_size\": 2048\n  },\n  \"collaborative_signal\": {\n    \"mode\": \"co_occurrence_contrastive\",\n    \"interactions\": {\n      \"uri\": \"s3://acme/sequences.jsonl\",\n      \"format\": \"jsonl\"\n    },\n    \"weight\": 0.3\n  },\n  \"progressive_masking\": true\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("Authorization", "Bearer <token>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://api.jeantechnologies.com/v1/tokenizers")
  .header("Authorization", "Bearer <token>")
  .header("Content-Type", "application/json")
  .body("{\n  \"name\": \"catalog-v1\",\n  \"catalog\": {\n    \"uri\": \"s3://acme/catalog.jsonl\",\n    \"format\": \"jsonl\"\n  },\n  \"modalities\": [\n    \"text\",\n    \"image\"\n  ],\n  \"codebook\": {\n    \"levels\": 4,\n    \"resolution\": \"multi\",\n    \"base_size\": 2048\n  },\n  \"collaborative_signal\": {\n    \"mode\": \"co_occurrence_contrastive\",\n    \"interactions\": {\n      \"uri\": \"s3://acme/sequences.jsonl\",\n      \"format\": \"jsonl\"\n    },\n    \"weight\": 0.3\n  },\n  \"progressive_masking\": true\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://api.jeantechnologies.com/v1/tokenizers")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n  \"name\": \"catalog-v1\",\n  \"catalog\": {\n    \"uri\": \"s3://acme/catalog.jsonl\",\n    \"format\": \"jsonl\"\n  },\n  \"modalities\": [\n    \"text\",\n    \"image\"\n  ],\n  \"codebook\": {\n    \"levels\": 4,\n    \"resolution\": \"multi\",\n    \"base_size\": 2048\n  },\n  \"collaborative_signal\": {\n    \"mode\": \"co_occurrence_contrastive\",\n    \"interactions\": {\n      \"uri\": \"s3://acme/sequences.jsonl\",\n      \"format\": \"jsonl\"\n    },\n    \"weight\": 0.3\n  },\n  \"progressive_masking\": true\n}"

response = http.request(request)
puts response.read_body

{
  "tokenizer_id": "tok_9k2m",
  "name": "catalog-v1",
  "status": "queued",
  "codebook": {
    "levels": 4,
    "resolution": "multi",
    "base_size": 2048
  },
  "created_at": "2026-08-31T18:04:11Z",
  "ready_at": null
}

{
  "error": {
    "code": "invalid_request",
    "message": "Field 'domain' is required.",
    "details": {}
  }
}

{
  "error": {
    "code": "invalid_request",
    "message": "Field 'domain' is required.",
    "details": {}
  }
}

The one call that matters. Everything downstream inherits the quality of this fit. Item content is encoded per modality and residually quantized into levels codebooks. Level 1 places the item in a coarse semantic region; each subsequent level refines within it. See Semantic IDs for the reasoning behind the defaults.

curl https://api.jeantechnologies.com/v1/tokenizers \
  -H "Authorization: Bearer $JEAN_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "name": "catalog-v1",
    "catalog": { "uri": "s3://acme/catalog.jsonl" },
    "modalities": ["text", "image"],
    "codebook": { "levels": 4, "resolution": "multi", "base_size": 2048 },
    "collaborative_signal": {
      "mode": "co_occurrence_contrastive",
      "interactions": { "uri": "s3://acme/sequences.jsonl" },
      "weight": 0.3
    },
    "progressive_masking": true
  }'

job = requests.post(
    "https://api.jeantechnologies.com/v1/tokenizers",
    headers={"Authorization": f"Bearer {os.environ['JEAN_API_KEY']}"},
    json={
        "name": "catalog-v1",
        "catalog": {"uri": "s3://acme/catalog.jsonl"},
        "codebook": {"levels": 4, "resolution": "multi", "base_size": 2048},
        "collaborative_signal": {
            "mode": "co_occurrence_contrastive",
            "interactions": {"uri": "s3://acme/sequences.jsonl"},
        },
    },
).json()

const res = await fetch("https://api.jeantechnologies.com/v1/tokenizers", {
  method: "POST",
  headers: {
    Authorization: `Bearer ${process.env.JEAN_API_KEY}`,
    "Content-Type": "application/json",
  },
  body: JSON.stringify({
    name: "catalog-v1",
    catalog: { uri: "s3://acme/catalog.jsonl" },
    codebook: { levels: 4, resolution: "multi", base_size: 2048 },
    collaborative_signal: {
      mode: "co_occurrence_contrastive",
      interactions: { uri: "s3://acme/sequences.jsonl" },
    },
  }),
});
const tokenizer = await res.json();

Input format

catalog is JSONL or Parquet, one item per record:

{"item_id": "sku_771", "title": "Merino crew socks", "description": "Mid-weight, charcoal.", "image_url": "https://acme.com/771.jpg", "attributes": {"brand": "Acme"}}

collaborative_signal.interactions is one user sequence per record, oldest first:

{"user_id": "u_18", "items": ["sku_310", "sku_884", "sku_771"], "timestamps": [1735689600, 1736294400, 1736899200]}

Choosing a codebook

resolution: "multi" sizes each level as base_size / 2^(level-1). With the defaults:

Level	Size	Encodes
1	2048	Coarse semantic region
2	1024	Sub-region
3	512	Fine distinction
4	256	Residual detail

Total addressable space is the product of the level sizes, so the defaults cover roughly 275 billion distinct IDs. The binding constraint is almost never address space. It is whether level 1 has enough distinct semantic regions in your catalog to fill 2048 codes.

Rule of thumb: start with base_size near the number of genuinely distinct categories in your catalog, rounded up to a power of two. Check codebook_utilization on the fitted tokenizer and adjust. Utilization below about 0.5 at level 1 means you oversized it.

resolution: "uniform" is available for parity with older RQ-VAE setups but is not recommended. By level 4 there is little residual entropy left to encode, and a full-width codebook there mostly sits unused.

Response

Fitting is asynchronous. The call returns 202 immediately:

{
  "tokenizer_id": "tok_9k2m",
  "name": "catalog-v1",
  "status": "queued",
  "codebook": { "levels": 4, "resolution": "multi", "base_size": 2048 },
  "created_at": "2026-08-31T18:04:11Z",
  "ready_at": null
}

Poll Get tokenizer until status is ready.

Authorizations

Authorization

string

header

required

API key issued by Jean Technologies. Contact the team for access.

Body

application/json

name

string

required

Example:

"catalog-v1"

catalog

object

required

Show child attributes

modalities

enum<string>[]

Which item content to encode. Each modality gets its own frozen encoder; outputs are concatenated before quantization.

Available options:

text,

image

codebook

object

Residual quantization layout.

Show child attributes

collaborative_signal

object

How behavioral signal shapes the quantizer.

Show child attributes

progressive_masking

boolean

default:true

Randomly truncate to the first r levels during training so each level has to be meaningful on its own rather than only in combination with deeper levels. This is what makes prefix-constrained beam search prune a coherent region at decode time.

Response

Fit queued

tokenizer_id

string

Example:

"tok_9k2m"

name

string

Example:

"catalog-v1"

status

enum<string>

Available options:

queued,

fitting,

ready,

failed

Example:

"ready"

codebook

object

Residual quantization layout.

Show child attributes

items_tokenized

integer

Example:

1840221

codebook_utilization

number[]

Fraction of each level's codebook in use, ordered by level.

Example:

[0.97, 0.91, 0.78, 0.61]

collision_rate

number

Fraction of items that required a disambiguating suffix to stay uniquely addressable.

Example:

0.004

created_at

string<date-time>

Example:

"2026-08-31T18:04:11Z"

ready_at

string<date-time> | null

Example:

"2026-08-31T19:22:47Z"

Semantic IDs

Users

Matching

Feedback & Training

Schema

Input format

Choosing a codebook

Response

Authorizations

Body

Response

​Input format

​Choosing a codebook

​Response

Authorizations

Body

Response

Input format

Choosing a codebook

Response