ocr-coreml-detector-224 / conversion_config.yaml
mweinbach1's picture
Upload CoreML detector package
b79acd1 verified
raw
history blame contribute delete
536 Bytes
name: detector_ane_decomposed_int8_768
description: ANE-decomposed detector at 768 px, int8 per-channel weight quantization.
model: { module: detector, checkpoint: v2_english }
detector_input:
batch_size: 1
infer_length: 768
ane_decomposed: true
conversion:
compute_precision: fp16
compute_units: CPU_AND_NE
minimum_deployment_target: iOS18
quantize_nbits: 8
quantize_granularity: per_channel
quantize_mode: linear_symmetric
compare:
atol: 2.0e+2
rtol: 5.0e-1
num_samples: 2
seed: 0
include_real_image: true