From 328944e714eccb6c1fd07508c8eca81552fa1d5b Mon Sep 17 00:00:00 2001 From: Viacheslav Klimkov Date: Tue, 2 Jun 2026 18:20:44 +0200 Subject: [PATCH 01/45] nemo/collections/tts/models/easy_magpietts_inference.py: remove duplicate speaker encoder application Signed-off-by: Viacheslav Klimkov --- .../tts/models/easy_magpietts_inference.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/nemo/collections/tts/models/easy_magpietts_inference.py b/nemo/collections/tts/models/easy_magpietts_inference.py index 67eb7212d6ea..d0951dc56d9a 100644 --- a/nemo/collections/tts/models/easy_magpietts_inference.py +++ b/nemo/collections/tts/models/easy_magpietts_inference.py @@ -958,22 +958,6 @@ def prepare_context_tensors( context_audio_embedded = self.embed_audio_tokens(context_audio_codes) # (B, T', E) batch_size = context_audio_embedded.size(0) - if self.use_speaker_encoder: - if ( - self.training - and batch_size > 1 - and self.train_shuffle_context_embedding_prob > 0 - and random.random() < self.train_shuffle_context_embedding_prob - ): - # Feed shuffled raw context embeddings (without speaker encoder) so - # the decoder cannot rely on direct unencoded speaker identity cues. - shift = random.randint(1, batch_size - 1) - context_audio_embedded = context_audio_embedded.roll(shift, dims=0) - else: - context_audio_embedded = self.encode_context_audio_embeddings( - context_audio_embedded=context_audio_embedded, context_audio_lens=context_audio_codes_lens - ) - if self.use_speaker_encoder: if ( self.training From 78404cbcf4dc114639b89ad43bce79390c25583e Mon Sep 17 00:00:00 2001 From: Viacheslav Klimkov Date: Tue, 2 Jun 2026 18:20:44 +0200 Subject: [PATCH 02/45] examples/tts/easymagpie_vllm_omni: initial commit for vllm_omni definition of Easy Magpie Signed-off-by: Viacheslav Klimkov --- ...easy_magpietts_extract_speaker_encoding.py | 164 +++++ .../easy_magpietts_single_infer.py | 141 +++++ .../easymagpie_inference_demo.ipynb | 419 +++++++++++++ .../easymagpie_vllm_omni/__init__.py | 27 + .../easymagpie_vllm_omni/config.py | 158 +++++ .../easymagpie_vllm_omni/easymagpie.py | 586 ++++++++++++++++++ .../easymagpie_vllm_omni/local_transformer.py | 398 ++++++++++++ .../tts/easymagpie_vllm_omni/pyproject.toml | 20 + .../vllm_plugin_easymagpie_omni/__init__.py | 51 ++ 9 files changed, 1964 insertions(+) create mode 100644 examples/tts/easymagpie_vllm_omni/easy_magpietts_extract_speaker_encoding.py create mode 100644 examples/tts/easymagpie_vllm_omni/easy_magpietts_single_infer.py create mode 100644 examples/tts/easymagpie_vllm_omni/easymagpie_inference_demo.ipynb create mode 100644 examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/__init__.py create mode 100644 examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/config.py create mode 100644 examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/easymagpie.py create mode 100644 examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/local_transformer.py create mode 100644 examples/tts/easymagpie_vllm_omni/pyproject.toml create mode 100644 examples/tts/easymagpie_vllm_omni/vllm_plugin_easymagpie_omni/__init__.py diff --git a/examples/tts/easymagpie_vllm_omni/easy_magpietts_extract_speaker_encoding.py b/examples/tts/easymagpie_vllm_omni/easy_magpietts_extract_speaker_encoding.py new file mode 100644 index 000000000000..90f47dbd6a49 --- /dev/null +++ b/examples/tts/easymagpie_vllm_omni/easy_magpietts_extract_speaker_encoding.py @@ -0,0 +1,164 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Standalone speaker-encoder output extractor for EasyMagpieTTS. + +Pre-computes ONLY the speaker-encoded context-audio embedding so it can be fed +to a separate (e.g. vLLM) backbone implementation. Context-text / task +embeddings are intentionally NOT included here -- the caller is expected to +prepend/append those (e.g. inside the vLLM model's ``preprocess``). + +This reproduces the audio branch of +``EasyMagpieTTSInferenceModel.prepare_context_tensors``:: + + audio -> codec codes -> (codec convert) -> add BOS/EOS -> frame stacking + -> per-codebook embedding -> speaker encoder + +and saves the resulting ``(T_audio, embedding_dim)`` tensor to disk. + +Example: + python examples/tts/easy_magpietts_extract_speaker_encoding.py \\ + --nemo_file /path/to/EMTTS_Pretraining_Qwen_WithCrossLingual_3_5_Delay.nemo \\ + --codec_model_path /path/to/25fps_spectral_codec_with_bandwidth_extension.nemo \\ + --phoneme_tokenizer_path /path/to/bpe_ipa_tokenizer_2048_en_de_es_fr_hi_it_vi_zh.json \\ + --context_audio /path/to/reference_voice.wav \\ + --out_file ./speaker_encoding.pt +""" +from __future__ import annotations + +import argparse + +import torch + +from nemo.collections.tts.modules.magpietts_inference.utils import ModelLoadConfig, load_easy_magpie_model +from nemo.collections.tts.modules.magpietts_modules import add_special_tokens +from nemo.utils import logging + + +def main(): + parser = argparse.ArgumentParser(description="Extract EasyMagpieTTS speaker-encoder output") + parser.add_argument("--nemo_file", required=True, help="Path to the EasyMagpieTTS .nemo checkpoint") + parser.add_argument("--codec_model_path", required=True, help="Path to the audio codec .nemo checkpoint") + parser.add_argument( + "--phoneme_tokenizer_path", + default=None, + help="Override the phoneme (IPA BPE) tokenizer path baked into the checkpoint. " + "Required if the path stored in the .nemo does not exist locally.", + ) + parser.add_argument("--context_audio", required=True, help="Reference/context wav for voice cloning") + parser.add_argument( + "--disable_cas_for_context_text", + action="store_true", + help="Set for legacy checkpoints trained without CAS embeddings on context text", + ) + parser.add_argument("--context_audio_duration", type=float, default=5.0) + parser.add_argument("--device", default="cuda") + parser.add_argument( + "--out_file", + default="./speaker_encoding.pt", + help="Output path. A torch .pt file (dict) is written; if it ends with .npy the " + "speaker-encoding tensor is saved as a NumPy array instead.", + ) + + args = parser.parse_args() + + model, ckpt_name = load_easy_magpie_model( + ModelLoadConfig( + nemo_file=args.nemo_file, + codecmodel_path=args.codec_model_path, + phoneme_tokenizer_path=args.phoneme_tokenizer_path, + disable_cas_for_context_text=args.disable_cas_for_context_text, + ), + device=args.device, + ) + logging.info(f"Loaded EasyMagpieTTS checkpoint: {ckpt_name}") + logging.info(f"use_speaker_encoder={getattr(model, 'use_speaker_encoder', False)}") + + device = next(model.parameters()).device + + with torch.inference_mode(): + # Load + trim context audio exactly like EasyMagpieTTSInferenceModel.do_tts. + context_audio = model._load_audio_for_inference(args.context_audio, model.sample_rate) + context_audio = model._adjust_audio_to_duration_for_inference( + context_audio, + model.sample_rate, + args.context_audio_duration, + model.codec_model_samples_per_frame, + ) + context_audio = context_audio.to(device) + context_audio_lens = torch.tensor([context_audio.size(1)], dtype=torch.long, device=device) + context_audio_codes, context_audio_codes_lens = model._codec_helper.audio_to_codes( + context_audio, context_audio_lens + ) + + # --- Audio branch of prepare_context_tensors (no context text / task embedding) --- + if model._codec_converter is not None: + context_audio_codes = model._codec_converter.convert_original_to_new( + audio_tokens=context_audio_codes, audio_lens=context_audio_codes_lens + ).long() + + context_audio_codes, context_audio_codes_lens = add_special_tokens( + codes=context_audio_codes, + codes_len=context_audio_codes_lens, + bos_id=model.context_audio_bos_id, + eos_id=model.context_audio_eos_id, + ) + + context_audio_codes, context_audio_codes_lens = model.stack_codes( + context_audio_codes, + context_audio_codes_lens, + model.context_audio_bos_id, + model.context_audio_eos_id, + model.frame_stacking_factor, + model.num_audio_codebooks, + ) + + context_audio_embedded = model.embed_audio_tokens(context_audio_codes) # (B, T_audio, E) + + if getattr(model, "use_speaker_encoder", False): + context_audio_embedded = model.encode_context_audio_embeddings( + context_audio_embedded=context_audio_embedded, + context_audio_lens=context_audio_codes_lens, + ) + else: + logging.warning( + "Checkpoint has use_speaker_encoder=False; saving raw per-codebook audio embeddings " + "(no speaker encoder applied)." + ) + + # Strip batch dim (B == 1) -> (T_audio, embedding_dim). + audio_len = int(context_audio_codes_lens[0].item()) + speaker_encoding = context_audio_embedded[0, :audio_len].contiguous().float().detach().cpu() + logging.info(f"Extracted speaker-encoder output: {tuple(speaker_encoding.shape)}") + + if args.out_file.endswith(".npy"): + import numpy as np + + np.save(args.out_file, speaker_encoding.numpy()) + else: + torch.save( + { + "speaker_encoding": speaker_encoding, + "context_audio": args.context_audio, + "embedding_dim": int(speaker_encoding.size(-1)), + "num_frames": int(speaker_encoding.size(0)), + "checkpoint": ckpt_name, + }, + args.out_file, + ) + logging.info(f"Wrote speaker encoding of shape {tuple(speaker_encoding.shape)} to {args.out_file}") + + +if __name__ == "__main__": + main() diff --git a/examples/tts/easymagpie_vllm_omni/easy_magpietts_single_infer.py b/examples/tts/easymagpie_vllm_omni/easy_magpietts_single_infer.py new file mode 100644 index 000000000000..313f4caa7f61 --- /dev/null +++ b/examples/tts/easymagpie_vllm_omni/easy_magpietts_single_infer.py @@ -0,0 +1,141 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Minimal pure-PyTorch single-utterance inference for EasyMagpieTTS. + +No vLLM, no manifest, no evalset config. Just: one context wav + one text -> one wav. + +Example: + python examples/tts/easy_magpietts_single_infer.py \\ + --nemo_file /path/to/EMTTS_Pretraining_Qwen_WithCrossLingual_3_5_Delay.nemo \\ + --codec_model_path /path/to/25fps_spectral_codec_with_bandwidth_extension.nemo \\ + --phoneme_tokenizer_path /path/to/bpe_ipa_tokenizer_2048_en_de_es_fr_hi_it_vi_zh.json \\ + --context_audio /path/to/reference_voice.wav \\ + --text "Hello, this is a test of the EasyMagpie text to speech model." \\ + --out_wav ./out.wav +""" +from __future__ import annotations + +import argparse + +import soundfile as sf +import torch + +from nemo.collections.tts.modules.magpietts_inference.utils import ModelLoadConfig, load_easy_magpie_model +from nemo.utils import logging + + +def main(): + parser = argparse.ArgumentParser(description="EasyMagpieTTS single-utterance pure-torch inference") + parser.add_argument("--nemo_file", required=True, help="Path to the EasyMagpieTTS .nemo checkpoint") + parser.add_argument("--codec_model_path", required=True, help="Path to the audio codec .nemo checkpoint") + parser.add_argument( + "--phoneme_tokenizer_path", + default=None, + help="Override the phoneme (IPA BPE) tokenizer path baked into the checkpoint. " + "Required if the path stored in the .nemo does not exist locally.", + ) + parser.add_argument("--context_audio", default=None, help="Reference/context wav for voice cloning") + parser.add_argument( + "--context_text", + default=None, + help="Optional style/context text tag. The voice is cloned from --context_audio; this is a " + "separate style/language conditioning string. If omitted, the correct in-distribution " + '"no text context" placeholder is auto-selected to match how the checkpoint was trained ' + "(language tag like [EN] if add_language_to_context_text=True, else [NO TEXT CONTEXT]). " + "Do NOT pass a free-form sentence unless you want it spoken/styled.", + ) + parser.add_argument( + "--language", + default="en", + help="Language of --text; used to build the [LANG] context-text placeholder for checkpoints " + "trained with add_language_to_context_text=True (e.g. en, de, es, fr, it, hi, zh, vi, ko-KR, pt-BR, ar)", + ) + parser.add_argument("--text", required=True, help="Text to synthesize") + parser.add_argument("--out_wav", default="./out.wav", help="Output wav path") + + # Tokenizer selection: defaults to the first text tokenizer in the checkpoint config + # (e.g. nemotron_nano_30b). Override only if your checkpoint has multiple. + parser.add_argument("--main_tokenizer_name", default=None) + + # The legacy Qwen EasyMagpie checkpoint was trained without CAS embeddings on context text. + parser.add_argument( + "--disable_cas_for_context_text", + action="store_true", + help="Set for legacy checkpoints trained without CAS embeddings on context text", + ) + + # Sampling / decoding parameters (defaults mirror the InferEvaluate functional test). + parser.add_argument("--temperature", type=float, default=0.6) + parser.add_argument("--topk", type=int, default=80) + parser.add_argument("--use_cfg", action="store_true", default=True) + parser.add_argument("--no_cfg", dest="use_cfg", action="store_false") + parser.add_argument("--cfg_scale", type=float, default=2.5) + parser.add_argument("--no_local_transformer", dest="use_local_transformer", action="store_false", default=True) + parser.add_argument("--max_steps", type=int, default=500) + parser.add_argument("--context_audio_duration", type=float, default=5.0) + parser.add_argument("--device", default="cuda") + + args = parser.parse_args() + + model, ckpt_name = load_easy_magpie_model( + ModelLoadConfig( + nemo_file=args.nemo_file, + codecmodel_path=args.codec_model_path, + phoneme_tokenizer_path=args.phoneme_tokenizer_path, + disable_cas_for_context_text=args.disable_cas_for_context_text, + ), + device=args.device, + ) + logging.info(f"Loaded EasyMagpieTTS checkpoint: {ckpt_name}") + logging.info(f"Available text tokenizers: {list(model.tokenizer.tokenizers.keys())}") + + # Resolve the context-text placeholder to match the training-time convention. + # The dataset uses "[]" when add_language_to_context_text=True, else "[NO TEXT CONTEXT]". + # Passing the wrong placeholder is out-of-distribution and the model may literally speak it + # (e.g. starting the audio with the word "context"). + context_text = args.context_text + if context_text is None: + if getattr(model, "add_language_to_context_text", False): + context_text = f"[{args.language.upper()}]" + else: + context_text = "[NO TEXT CONTEXT]" + logging.info(f"Using context_text={context_text!r}") + + with torch.inference_mode(): + audio, audio_lens = model.do_tts( + transcript=args.text, + context_audio_file_path=args.context_audio, + context_text=context_text, + main_tokenizer_name=args.main_tokenizer_name, + context_audio_duration=args.context_audio_duration, + use_cfg=args.use_cfg, + cfg_scale=args.cfg_scale, + use_local_transformer=args.use_local_transformer, + temperature=args.temperature, + topk=args.topk, + max_steps=args.max_steps, + ) + + audio_len = int(audio_lens[0].item()) + audio_np = audio[0, :audio_len].float().detach().cpu().numpy() + sf.write(args.out_wav, audio_np, model.output_sample_rate) + logging.info( + f"Wrote {audio_len / model.output_sample_rate:.2f}s of audio " + f"({model.output_sample_rate} Hz) to {args.out_wav}" + ) + + +if __name__ == "__main__": + main() diff --git a/examples/tts/easymagpie_vllm_omni/easymagpie_inference_demo.ipynb b/examples/tts/easymagpie_vllm_omni/easymagpie_inference_demo.ipynb new file mode 100644 index 000000000000..0e2693776adf --- /dev/null +++ b/examples/tts/easymagpie_vllm_omni/easymagpie_inference_demo.ipynb @@ -0,0 +1,419 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "d5a1129d", + "metadata": {}, + "source": [ + "# EasyMagpieTTS — vLLM-Omni inference demo (dummy weights)\n", + "\n", + "This notebook runs an end-to-end inference pass through the\n", + "[`easymagpie_vllm_omni`](./easymagpie_vllm_omni) model definition using\n", + "**dummy / random weights**, so you can exercise the full engine path\n", + "(prefill -> autoregressive decode -> audio-code extraction) without a converted\n", + "checkpoint.\n", + "\n", + "It follows the same `AsyncOmni` single-stage pattern as the reference\n", + "`qwen3-tts` and `eartts` demos:\n", + "\n", + "* **prefill** — the caller supplies a precomputed context embedding via\n", + " `additional_information.prompt_embeds` of shape `(T_ctx, embedding_dim)`, with\n", + " `prompt_token_ids = [0] * T_ctx` (exactly like qwen3-tts `talker_prompt_embeds`\n", + " / eartts `speaker_latent`).\n", + "* **decode** — each step consumes one subword id from the streaming\n", + " `additional_information.text_tokens` list; the local transformer samples all\n", + " `C * S` stacked audio codebooks for the frame.\n", + "* **output** — per-step audio codes are surfaced on\n", + " `OmniOutput.multimodal_outputs[\\\"audio_codes\\\"]` (`BT x num_codebooks`), and the\n", + " engine accumulates them across steps just like eartts, so we trim to the last\n", + " `len(token_ids)` decoded rows.\n", + "\n", + "> **Dummy weights.** We build a tiny `config.json` (small backbone + small\n", + "> codebooks) and start the engine with `load_format=\\\"dummy\\\"`, so vLLM fills all\n", + "> parameters with random values. The emitted codes are therefore meaningless —\n", + "> this is a *smoke test* of the engine wiring, not a real synthesis. Point the\n", + "> engine at a real converted checkpoint (and drop `load_format`) to get audio.\n", + "\n", + "> **Environment.** Run this inside the bootstrapped `vllm_omni_env` (vLLM +\n", + "> vLLM-Omni + compatible torch) with the plugin installed:\n", + "> ```bash\n", + "> source /path/to/vllm_omni_env/bin/activate\n", + "> pip install -e examples/tts/easymagpie_vllm_omni\n", + "> ```" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c9a71b74", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "# Single-process executor below, but keep spawn semantics consistent with the\n", + "# qwen3-tts / eartts demos in case you switch to a multiproc backend.\n", + "os.environ.setdefault(\"VLLM_WORKER_MULTIPROC_METHOD\", \"spawn\")\n", + "\n", + "import json\n", + "import tempfile\n", + "import uuid\n", + "from pathlib import Path\n", + "\n", + "import torch\n", + "import yaml\n", + "\n", + "from vllm import SamplingParams\n", + "from vllm_omni import AsyncOmni\n", + "\n", + "# Importing the model package is optional (the engine resolves the arch via the\n", + "# `vllm.general_plugins` entry point installed with the package), but doing it\n", + "# here surfaces the arch dataclass we use to size the dummy prompt embedding.\n", + "from easymagpie_vllm_omni.config import EasyMagpieOmniArch\n", + "\n", + "print(\"torch:\", torch.__version__, \"| cuda:\", torch.cuda.is_available())" + ] + }, + { + "cell_type": "markdown", + "id": "f7ff55fe", + "metadata": {}, + "source": [ + "## 1. Build a tiny dummy model directory\n", + "\n", + "The engine only needs a `config.json` that (a) names the registered arch and\n", + "(b) carries the EasyMagpie + Qwen2 scalars. We deliberately pick **small** dims\n", + "so the dummy backbone and local transformer are fast to instantiate.\n", + "\n", + "The EasyMagpie-specific scalars (`embedding_dim`, `num_audio_codebooks`,\n", + "`codebook_size`, `frame_stacking_factor`, `local_transformer_*`, …) are read by\n", + "`EasyMagpieOmniArch.from_hf_config`; the standard Qwen2 fields (`hidden_size`,\n", + "`num_hidden_layers`, …) configure the reused `Qwen2Model` backbone. Setting\n", + "`phoneme_vocab_size = 0` disables the optional phoneme branch for simplicity.\n", + "\n", + "With `load_format=\\\"dummy\\\"` (set in the stage config) vLLM never reads weight\n", + "files, so a lone `config.json` is enough — no safetensors, no tokenizer." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3e0df89e", + "metadata": {}, + "outputs": [], + "source": [ + "# Small, internally-consistent dummy profile.\n", + "# embedding_dim == hidden_size == audio_embedding_dim == local_transformer_hidden_dim\n", + "# keeps every in/out projection an Identity (fewer dummy params, same code path).\n", + "HIDDEN = 256\n", + "NUM_AUDIO_CODEBOOKS = 4\n", + "CODEBOOK_SIZE = 64\n", + "FRAME_STACKING = 2 # -> num_stacked_codebooks = NUM_AUDIO_CODEBOOKS * FRAME_STACKING = 8\n", + "TEXT_VOCAB = 256\n", + "\n", + "config = {\n", + " # Resolved through the `vllm.general_plugins` entry point registered by the\n", + " # `easymagpie_vllm_omni` package -> EasyMagpieTTSForConditionalGeneration.\n", + " \"architectures\": [\"EasyMagpieTTSForConditionalGeneration\"],\n", + " # Standard Qwen2 backbone fields (consumed by vllm Qwen2Model).\n", + " \"model_type\": \"qwen2\",\n", + " \"hidden_size\": HIDDEN,\n", + " \"intermediate_size\": 4 * HIDDEN,\n", + " \"num_hidden_layers\": 2,\n", + " \"num_attention_heads\": 4,\n", + " \"num_key_value_heads\": 4,\n", + " \"max_position_embeddings\": 4096,\n", + " \"rms_norm_eps\": 1e-6,\n", + " \"rope_theta\": 1000000.0,\n", + " \"vocab_size\": TEXT_VOCAB,\n", + " \"tie_word_embeddings\": False,\n", + " \"torch_dtype\": \"float32\",\n", + " # EasyMagpie-specific scalars (read by EasyMagpieOmniArch.from_hf_config).\n", + " \"text_vocab_size\": TEXT_VOCAB,\n", + " \"embedding_dim\": HIDDEN,\n", + " \"audio_embedding_dim\": HIDDEN,\n", + " \"num_audio_codebooks\": NUM_AUDIO_CODEBOOKS,\n", + " \"codebook_size\": CODEBOOK_SIZE,\n", + " \"frame_stacking_factor\": FRAME_STACKING,\n", + " \"phoneme_stacking_factor\": 0, # disable phoneme branch\n", + " \"phoneme_vocab_size\": 0,\n", + " \"local_transformer_n_layers\": 2,\n", + " \"local_transformer_n_heads\": 4,\n", + " \"local_transformer_hidden_dim\": HIDDEN,\n", + "}\n", + "\n", + "MODEL_DIR = Path(tempfile.mkdtemp(prefix=\"easymagpie_dummy_\"))\n", + "(MODEL_DIR / \"config.json\").write_text(json.dumps(config, indent=2))\n", + "print(f\"Dummy model dir: {MODEL_DIR}\")\n", + "\n", + "# Sanity-check the arch the model will derive from this config.\n", + "arch = EasyMagpieOmniArch.from_hf_config(type(\"Cfg\", (), config))\n", + "print(f\"embedding_dim : {arch.embedding_dim}\")\n", + "print(f\"num_stacked_codebooks : {arch.num_stacked_codebooks} (C*S)\")\n", + "print(f\"tokens / codebook : {arch.num_all_tokens_per_codebook} (codebook_size + specials)\")\n", + "print(f\"audio_bos / audio_eos id : {arch.audio_bos_id} / {arch.audio_eos_id}\")" + ] + }, + { + "cell_type": "markdown", + "id": "012df58d", + "metadata": {}, + "source": [ + "## 2. Single-stage `AsyncOmni` engine\n", + "\n", + "A single `llm` stage that runs the EasyMagpie talker, mirroring the eartts demo\n", + "(`worker_type=\\\"ar\\\"`, `OmniARScheduler`). The stage declares\n", + "`engine_output_type=\\\"audio\\\"` / `final_output_type=\\\"audio\\\"`: for a single-stage\n", + "AR TTS model these make the runner attach the per-step `audio_codes` multimodal\n", + "payload to the output (with `\\\"latent\\\"` the payload is dropped because nothing\n", + "downstream consumes it, and `multimodal_output[\\\"audio_codes\\\"]` comes back\n", + "`None`). Two extra knobs make this a dummy-weights run with no external assets:\n", + "\n", + "* `load_format: \\\"dummy\\\"` — vLLM initializes random weights instead of reading a\n", + " checkpoint (so `load_weights` / `init_forbidden_mask` are skipped; the\n", + " forbidden-token mask stays all-zeros, i.e. no sampling mask — fine for a smoke\n", + " test).\n", + "* `skip_tokenizer_init: true` — we feed `prompt_token_ids` + `text_tokens`\n", + " directly, so no tokenizer files are needed.\n", + "\n", + "`max_model_len` must cover `T_ctx` (prefill) + the number of decode steps." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5085e9a4", + "metadata": {}, + "outputs": [], + "source": [ + "T_CTX = 16 # prefill context-embedding length (prompt_token_ids = [0] * T_CTX)\n", + "DECODE_STEPS = 32 # number of audio frames to decode\n", + "MAX_MODEL_LEN = 512\n", + "MAX_NUM_BATCHED_TOKENS = 512\n", + "\n", + "stage_cfg = {\n", + " \"stage_args\": [\n", + " {\n", + " \"stage_id\": 0,\n", + " \"stage_type\": \"llm\",\n", + " \"is_comprehension\": True,\n", + " \"final_output\": True,\n", + " # \"audio\" (not \"latent\") is required for a single-stage AR TTS model:\n", + " # it makes the AR model runner attach the per-step multimodal payload\n", + " # (\"audio_codes\") to the EngineCoreOutput even though no downstream\n", + " # stage consumes it, so the codes reach the client. With \"latent\" the\n", + " # payload is dropped and multimodal_output[\"audio_codes\"] is None.\n", + " \"final_output_type\": \"audio\",\n", + " \"runtime\": {\"devices\": \"0\"},\n", + " \"engine_args\": {\n", + " \"model_stage\": \"easymagpie\",\n", + " \"max_num_seqs\": 1,\n", + " \"model_arch\": \"EasyMagpieTTSForConditionalGeneration\",\n", + " \"worker_type\": \"ar\",\n", + " \"scheduler_cls\": \"vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler\",\n", + " \"enforce_eager\": True, # dummy run: skip CUDA-graph capture for a faster start\n", + " \"trust_remote_code\": True,\n", + " \"async_scheduling\": True,\n", + " \"enable_prefix_caching\": False,\n", + " \"engine_output_type\": \"audio\",\n", + " \"gpu_memory_utilization\": 0.6,\n", + " \"distributed_executor_backend\": \"uni\",\n", + " \"max_num_batched_tokens\": MAX_NUM_BATCHED_TOKENS,\n", + " \"max_model_len\": MAX_MODEL_LEN,\n", + " \"dtype\": \"float32\",\n", + " \"attention_backend\": \"TRITON_ATTN\",\n", + " # --- dummy-weights smoke-test knobs ---\n", + " \"load_format\": \"dummy\",\n", + " \"skip_tokenizer_init\": True,\n", + " },\n", + " \"default_sampling_params\": {\n", + " \"temperature\": 0.0,\n", + " \"max_tokens\": DECODE_STEPS,\n", + " \"detokenize\": False,\n", + " \"ignore_eos\": True,\n", + " },\n", + " }\n", + " ],\n", + "}\n", + "\n", + "_tmp = tempfile.NamedTemporaryFile(\n", + " mode=\"w\", suffix=\".yaml\", prefix=\"easymagpie_omni_demo_\", delete=False,\n", + ")\n", + "yaml.dump(stage_cfg, _tmp, sort_keys=False)\n", + "_tmp.close()\n", + "STAGE_CFG_PATH = _tmp.name\n", + "print(f\"Stage config: {STAGE_CFG_PATH}\")\n", + "\n", + "omni = AsyncOmni(\n", + " model=str(MODEL_DIR),\n", + " stage_configs_path=STAGE_CFG_PATH,\n", + " log_stats=False,\n", + " stage_init_timeout=300,\n", + ")\n", + "print(\"Engine ready (single stage: EasyMagpie talker, dummy weights)\")" + ] + }, + { + "cell_type": "markdown", + "id": "2736b86d", + "metadata": {}, + "source": [ + "## 3. Build the prompt\n", + "\n", + "Two pieces of per-request input, passed through `additional_information`:\n", + "\n", + "* **`prompt_embeds`** `(T_ctx, embedding_dim)` — the precomputed context\n", + " embedding consumed during prefill. In a real run this is the speaker-encoded\n", + " context audio + context text produced by the caller; here we use random noise.\n", + " `prompt_token_ids = [0] * T_ctx` are placeholders (the model feeds the backbone\n", + " via `inputs_embeds`, never via these ids).\n", + "* **`text_tokens`** `list[int]` — the streaming subword stream; decode step `k`\n", + " consumes `text_tokens[k]`. We provide one id per decode step.\n", + "\n", + "(If the checkpoint had a phoneme branch you'd also stream `phoneme_tokens`; it's\n", + "disabled here via `phoneme_vocab_size = 0`.)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "697c74b3", + "metadata": {}, + "outputs": [], + "source": [ + "torch.manual_seed(0)\n", + "\n", + "# Precomputed context embedding (random stand-in for the speaker/text encoder).\n", + "prompt_embeds = torch.randn(T_CTX, arch.embedding_dim, dtype=torch.float32)\n", + "\n", + "# Streaming subword ids: one per decode step (step k consumes text_tokens[k]).\n", + "text_tokens = torch.randint(0, TEXT_VOCAB, (DECODE_STEPS,)).tolist()\n", + "\n", + "additional_information = {\n", + " \"prompt_embeds\": prompt_embeds, # (T_ctx, embedding_dim) tensor\n", + " \"text_tokens\": text_tokens, # list[int], grows by one per step\n", + "}\n", + "\n", + "prompt = {\n", + " \"prompt_token_ids\": [0] * T_CTX, # prefill placeholders\n", + " \"additional_information\": additional_information,\n", + "}\n", + "\n", + "sampling_params = SamplingParams(\n", + " temperature=0.0,\n", + " max_tokens=DECODE_STEPS,\n", + " detokenize=False,\n", + " ignore_eos=True, # dummy logits never emit a meaningful EOS -> run the full budget\n", + ")\n", + "\n", + "print(f\"T_ctx (prefill placeholders) : {T_CTX}\")\n", + "print(f\"prompt_embeds : {tuple(prompt_embeds.shape)}\")\n", + "print(f\"decode steps (max_tokens) : {DECODE_STEPS}\")\n", + "print(f\"text_tokens[:8] : {text_tokens[:8]}\")" + ] + }, + { + "cell_type": "markdown", + "id": "3ef8934d", + "metadata": {}, + "source": [ + "## 4. Run inference and extract audio codes\n", + "\n", + "`omni.generate(...)` is an async generator yielding one `RequestOutput` per\n", + "engine step; we keep the last one. As in the eartts demo, the accumulated\n", + "`multimodal_output[\\\"audio_codes\\\"]` holds one row per flat-batch token over the\n", + "whole run (the `T_ctx` prefill frames — codes left zero — plus one frame per\n", + "decode step), so we trim to the last `len(token_ids)` rows to recover just the\n", + "decoded frames." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6d0ccbd4", + "metadata": {}, + "outputs": [], + "source": [ + "async def run_request(prompt: dict, sampling_params):\n", + " request_id = f\"easymagpie-{uuid.uuid4().hex[:8]}\"\n", + " final_ro = None\n", + " num_steps = 0\n", + " async for stage_output in omni.generate(\n", + " prompt,\n", + " sampling_params_list=[sampling_params],\n", + " request_id=request_id,\n", + " ):\n", + " final_ro = stage_output\n", + " num_steps += 1\n", + " return final_ro, num_steps\n", + "\n", + "\n", + "final_ro, num_steps = await run_request(prompt, sampling_params)\n", + "assert final_ro is not None, \"no output from engine\"\n", + "\n", + "mm = final_ro.multimodal_output or {}\n", + "audio_codes = mm.get(\"audio_codes\")\n", + "token_ids = final_ro.outputs[0].token_ids if final_ro.outputs else []\n", + "\n", + "print(f\"Engine steps yielded : {num_steps}\")\n", + "print(f\"Layer-0 tokens (token_ids) : {len(token_ids)}\")\n", + "if isinstance(audio_codes, torch.Tensor):\n", + " audio_codes = audio_codes.detach().cpu().to(torch.long)\n", + " print(f\"audio_codes shape (raw) : {tuple(audio_codes.shape)}\")\n", + " # Trim the Tref prefill frames echoed during prefill: keep only the decoded\n", + " # frames (the last len(token_ids) rows), exactly like the eartts demo.\n", + " if len(token_ids) > 0:\n", + " audio_codes = audio_codes[-len(token_ids):].contiguous()\n", + " print(f\"audio_codes shape (decode) : {tuple(audio_codes.shape)}\")\n", + " print(f\"audio_codes dtype : {audio_codes.dtype}\")\n", + " print(f\"codes min / max : {int(audio_codes.min())} / {int(audio_codes.max())}\")\n", + " print(f\"first frame codes : {audio_codes[0].tolist()}\")\n", + "else:\n", + " print(f\"audio_codes : {audio_codes!r}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "04196662", + "metadata": {}, + "outputs": [], + "source": [ + "import matplotlib.pylab as plt\n", + "\n", + "plt.imshow(audio_codes.T, aspect=\"auto\")\n", + "plt.colorbar()\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3a6603b9", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "emp", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.13" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/__init__.py b/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/__init__.py new file mode 100644 index 000000000000..8a37af8454ea --- /dev/null +++ b/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/__init__.py @@ -0,0 +1,27 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""EasyMagpieTTS model definition for vLLM-Omni. + +This package provides an inference-only re-implementation of EasyMagpieTTS +(decoder-only, Qwen2 backbone + autoregressive local transformer over the +stacked audio codebooks) that plugs into the vLLM-Omni serving stack via the +standard ``preprocess`` / ``postprocess`` / ``make_omni_output`` hooks. + +The companion ``vllm_plugin_easymagpie_omni`` package registers the model with +vLLM's ``ModelRegistry`` through the ``vllm.general_plugins`` entry point. +""" + +from easymagpie_vllm_omni.config import EASYMAGPIE_QWEN, EasyMagpieOmniArch + +__all__ = ["EASYMAGPIE_QWEN", "EasyMagpieOmniArch"] diff --git a/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/config.py b/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/config.py new file mode 100644 index 000000000000..c569089e32f7 --- /dev/null +++ b/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/config.py @@ -0,0 +1,158 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Architecture constants for the EasyMagpieTTS vLLM-Omni model. + +These mirror the values baked into the reference EasyMagpieTTS checkpoint +(``examples/tts/conf/magpietts/easy_magpietts.yaml`` — Qwen2.5-1.5B backbone, +8 codebooks, frame-stacking ×2, 3-layer autoregressive local transformer). + +The vLLM-Omni model reads the bulk of its configuration from the +``hf_config`` provided by vLLM at construction time; this dataclass captures +the TTS-specific scalars that are *not* part of a standard HF text-LM config +and provides a single, well-documented default profile. +""" +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + +# Number of trailing special tokens appended to every audio codebook. +# Matches ``len(SpecialAudioToken)`` in +# ``nemo.collections.tts.modules.magpietts_modules`` (BOS, EOS, CONTEXT_BOS, +# CONTEXT_EOS, MASK, RESERVED_1..3). +NUM_SPECIAL_AUDIO_TOKENS: int = 8 + +# Offsets of the special audio tokens *within* the trailing special-token block +# (i.e. ``codebook_size + `` is the real embedding-table id). +SPECIAL_AUDIO_BOS: int = 0 +SPECIAL_AUDIO_EOS: int = 1 +SPECIAL_AUDIO_CONTEXT_BOS: int = 2 +SPECIAL_AUDIO_CONTEXT_EOS: int = 3 +SPECIAL_AUDIO_MASK: int = 4 + + +@dataclass +class EasyMagpieOmniArch: + """Static architecture description for an EasyMagpieTTS checkpoint. + + Attributes: + hidden_dim: Backbone hidden size (``cfg.hidden_dim``). + embedding_dim: Embedding size feeding the backbone (``cfg.embedding_dim``). + audio_embedding_dim: Per-codebook audio embedding size + (``cfg.audio_embedding_dim``); may differ from ``embedding_dim``. + num_audio_codebooks: Number of codec codebooks (``C``). + codebook_size: Base codec codebook size (excluding special tokens). + frame_stacking_factor: Frame stacking factor (``S``). The model treats + the audio stream as ``C * S`` independent "stacked" codebooks. + phoneme_stacking_factor: Phoneme stacking factor. + phoneme_vocab_size: Phoneme tokenizer vocabulary size. + local_transformer_n_layers / _n_heads / _hidden_dim: local-transformer + (intra-frame codebook predictor) sizing. + """ + + hidden_dim: int = 1536 + embedding_dim: int = 1536 + audio_embedding_dim: int = 1536 + + num_audio_codebooks: int = 8 + codebook_size: int = 1024 + frame_stacking_factor: int = 2 + + phoneme_stacking_factor: int = 1 + phoneme_vocab_size: int = 2051 + + local_transformer_n_layers: int = 3 + local_transformer_n_heads: int = 12 + local_transformer_hidden_dim: int = 1536 + + # Optional per-checkpoint overrides for backward compatibility (legacy + # checkpoints sometimes forced special-token ids). + forced_audio_bos_id: int | None = None + forced_audio_eos_id: int | None = None + forced_mask_token_id: int | None = None + + extra: dict[str, Any] = field(default_factory=dict) + + # ── Derived quantities ─────────────────────────────────────────── + @property + def num_stacked_codebooks(self) -> int: + """Number of independent codebooks the model autoregresses over (``C * S``).""" + return self.num_audio_codebooks * self.frame_stacking_factor + + @property + def num_all_tokens_per_codebook(self) -> int: + """Per-codebook vocabulary size including the trailing special tokens.""" + return self.codebook_size + NUM_SPECIAL_AUDIO_TOKENS + + @property + def audio_bos_id(self) -> int: + """Embedding-table id of the audio BOS token.""" + if self.forced_audio_bos_id is not None: + return self.forced_audio_bos_id + return self.codebook_size + SPECIAL_AUDIO_BOS + + @property + def audio_eos_id(self) -> int: + """Embedding-table id of the audio EOS token.""" + if self.forced_audio_eos_id is not None: + return self.forced_audio_eos_id + return self.codebook_size + SPECIAL_AUDIO_EOS + + @property + def mask_token_id(self) -> int: + """Embedding-table id of the MaskGit MASK token.""" + if self.forced_mask_token_id is not None: + return self.forced_mask_token_id + return self.codebook_size + SPECIAL_AUDIO_MASK + + @classmethod + def from_hf_config(cls, hf_config: Any) -> "EasyMagpieOmniArch": + """Build an arch description from a vLLM ``hf_config``. + + Any attribute present on ``hf_config`` overrides the default profile; + unknown attributes are ignored. This lets a converted checkpoint carry + its own ``easymagpie`` block in ``config.json`` while still working + out-of-the-box on the reference Qwen2.5-1.5B profile. + """ + defaults = cls() + kwargs: dict[str, Any] = {} + for f in ( + "hidden_dim", + "embedding_dim", + "audio_embedding_dim", + "num_audio_codebooks", + "codebook_size", + "frame_stacking_factor", + "phoneme_stacking_factor", + "phoneme_vocab_size", + "local_transformer_n_layers", + "local_transformer_n_heads", + "local_transformer_hidden_dim", + "forced_audio_bos_id", + "forced_audio_eos_id", + "forced_mask_token_id", + ): + if hasattr(hf_config, f): + kwargs[f] = getattr(hf_config, f) + # ``hidden_size`` is the canonical HF name for the backbone width. + if "hidden_dim" not in kwargs and hasattr(hf_config, "hidden_size"): + kwargs["hidden_dim"] = hf_config.hidden_size + kwargs.setdefault("embedding_dim", hf_config.hidden_size) + merged = {**defaults.__dict__, **kwargs} + merged.pop("extra", None) + return cls(**merged) + + +# Reference profile: Qwen2.5-1.5B backbone EasyMagpieTTS checkpoint. +EASYMAGPIE_QWEN = EasyMagpieOmniArch() diff --git a/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/easymagpie.py b/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/easymagpie.py new file mode 100644 index 000000000000..bfb76ccb9303 --- /dev/null +++ b/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/easymagpie.py @@ -0,0 +1,586 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Inference-only EasyMagpieTTS model for vLLM-Omni. + +EasyMagpieTTS is a decoder-only streaming TTS model: a text-LM backbone (the +reference checkpoint uses Qwen2.5-1.5B) consumes a per-frame additive input +embedding (text + phoneme + audio) and emits a per-frame hidden state, from +which a small autoregressive *local transformer* samples all ``C * S`` stacked +audio codebooks for that frame (see :mod:`easymagpie_vllm_omni.local_transformer`). + +This module wires that architecture into vLLM-Omni's +``preprocess`` / ``forward`` / ``compute_logits`` / ``make_omni_output`` / +``postprocess`` contract, following the same conventions as the upstream +qwen3-tts and eartts vLLM-Omni model definitions: + +* **Backbone** — vLLM's :class:`~vllm.model_executor.models.qwen2.Qwen2Model`, + reused wholesale (KV cache + paged attention) the same way the EasyMagpie + vLLM *sidecar* reuses ``NemotronHModel``. Every step feeds the backbone via + ``inputs_embeds``; its own ``embed_tokens`` table is never consumed. +* **Local transformer** — :class:`EasyMagpieCodePredictor`, a from-scratch, + CUDA-graph-capturable re-implementation that runs as a single compiled graph. +* **compute_logits** — returns trivial logits (à la eartts) so vLLM's sampler + always picks index 0; the real audio output is the codes tensor surfaced + through :meth:`make_omni_output` under the ``"audio_codes"`` key. + +Text is embedded via a precomputed per-subword lookup table baked at +checkpoint-conversion time (the reference char-aware subword encoder is +deterministic per subword id, so it is never run inside the engine). + +Per-request I/O (via ``additional_information``): + +* ``prompt_embeds`` (prefill only) — ``(T_ctx, embedding_dim)`` precomputed + context/prompt embedding (speaker-encoded context audio + context text) + produced by the caller, exactly like qwen3-tts ``talker_prompt_embeds`` / + eartts ``speaker_latent``. The user passes ``prompt_token_ids = [0] * T_ctx``. +* ``text_tokens`` — Python ``list[int]`` of subword ids that grows by one per + decode step; step ``k`` consumes ``text_tokens[k]`` (embedded through the + precomputed per-subword table). +* ``phoneme_tokens`` (optional) — same streaming-list contract for the phoneme + channel; if omitted the phoneme branch is skipped. +""" +from __future__ import annotations + +import bisect +from collections.abc import Iterable +from typing import Any, Optional + +import torch +from torch import nn +from vllm.compilation.backends import set_model_tag +from vllm.compilation.decorators import ignore_torch_compile, support_torch_compile +from vllm.config import CUDAGraphMode, VllmConfig +from vllm.forward_context import BatchDescriptor, get_forward_context +from vllm.logger import init_logger +from vllm.model_executor.models.qwen2 import Qwen2Model +from vllm.model_executor.models.utils import maybe_prefix +from vllm.sequence import IntermediateTensors + +from vllm_omni.model_executor.models.output_templates import OmniOutput + +from easymagpie_vllm_omni.config import EasyMagpieOmniArch +from easymagpie_vllm_omni.local_transformer import EasyMagpieCodePredictor + +logger = init_logger(__name__) + +# Placeholder token id stuffed into the per-step ``input_ids`` returned by +# ``preprocess`` — the model never consumes ``input_ids`` (decode behaviour is +# driven by the per-token buffers), and ``compute_logits`` returns +# argmax-at-0 dummy logits, so this only needs to be a valid id. +_DUMMY_TOKEN_ID = 0 + + +# ``dynamic_arg_dims`` is passed explicitly: this file uses +# ``from __future__ import annotations`` (PEP 563), so ``forward``'s annotations +# are strings and vLLM's annotation-based inference would fail with +# "No dynamic dimensions found...". These mirror vLLM's default inference +# (dim 0 for every tensor / IntermediateTensors argument). +@ignore_torch_compile +@support_torch_compile( + dynamic_arg_dims={ + "input_ids": 0, + "positions": 0, + "intermediate_tensors": 0, + "inputs_embeds": 0, + } +) +class EasyMagpieTTSForConditionalGeneration(nn.Module): + """EasyMagpieTTS talker for vLLM-Omni. + + See the module docstring for the per-step flow and the per-request I/O + contract. The class exposes the omni hooks (``has_preprocess`` / + ``has_postprocess`` / ``have_multimodal_outputs``) consumed by the + ``OmniGPUModelRunner``. + """ + + # Omni runner hooks. + has_preprocess: bool = True + has_postprocess: bool = True + have_multimodal_outputs: bool = True + + # Keep small per-step tensors GPU-resident across steps (no D2H/H2D). + gpu_resident_buffer_keys: set[str] = { + "last_audio_codes", + "last_phoneme_token", + "last_hidden", + } + + def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: + super().__init__() + hf_config = vllm_config.model_config.hf_config + self.hf_config = hf_config + self.vllm_config = vllm_config + self.arch = EasyMagpieOmniArch.from_hf_config(hf_config) + self.model_path = vllm_config.model_config.model + + arch = self.arch + self.hidden_dim = arch.hidden_dim + self.embedding_dim = arch.embedding_dim + self.num_codebooks = arch.num_stacked_codebooks + + # ── Backbone (reused vLLM text LM; fed via inputs_embeds) ─────── + self.backbone = Qwen2Model( + vllm_config=vllm_config, + prefix=maybe_prefix(prefix, "backbone"), + ) + + # ── Local transformer (its own compile group / CUDA graph) ────── + with set_model_tag("local_transformer"): + self.code_predictor = EasyMagpieCodePredictor( + vllm_config=vllm_config, + prefix=maybe_prefix(prefix, "code_predictor"), + ) + + # ── Text + phoneme embedding heads ────────────────────────────── + # Precomputed per-subword text embedding. The reference model embeds + # text with a char-aware subword (CAS) encoder + the decoder's subword + # table; both are deterministic per subword id, so the checkpoint + # converter bakes their combined result into this single lookup table + # (one row per subword id). It is fed additively on every decode step; + # the CAS encoder is never run inside the engine. + text_vocab_size = int(getattr(hf_config, "text_vocab_size", getattr(hf_config, "vocab_size", 0))) + self.text_embedding = nn.Embedding(text_vocab_size, self.embedding_dim) + + # Phoneme channel (optional — only built when the checkpoint has one). + self.has_phoneme = arch.phoneme_vocab_size > 0 and arch.phoneme_stacking_factor > 0 + if self.has_phoneme: + self.phoneme_embeddings = nn.ModuleList( + [nn.Embedding(arch.phoneme_vocab_size, self.embedding_dim) for _ in range(arch.phoneme_stacking_factor)] + ) + self.phoneme_final_proj = nn.Linear( + self.hidden_dim, arch.phoneme_vocab_size * arch.phoneme_stacking_factor + ) + + # ── Persistent, address-stable scratch buffers ───────────────── + max_num_tokens = vllm_config.scheduler_config.max_num_batched_tokens + dtype = vllm_config.model_config.dtype + # Combined per-token input embedding fed into the backbone. + self._combined_embeddings = torch.zeros(max_num_tokens, self.embedding_dim, dtype=dtype) + # Per-token decode inputs assembled by ``preprocess``. + self._dec_text_tokens = torch.zeros(max_num_tokens, dtype=torch.long) + self._dec_text_mask = torch.zeros(max_num_tokens, dtype=torch.long) + self._dec_audio_codes = torch.zeros(max_num_tokens, self.num_codebooks, dtype=torch.long) + self._dec_audio_valid = torch.zeros(max_num_tokens, dtype=torch.long) + if self.has_phoneme: + self._dec_phoneme_tokens = torch.zeros( + max_num_tokens, arch.phoneme_stacking_factor, dtype=torch.long + ) + self._dec_phoneme_valid = torch.zeros(max_num_tokens, dtype=torch.long) + + self._out_codes = torch.zeros(max_num_tokens, self.num_codebooks, dtype=torch.long) + + # ------------------------------------------------------------------ + # Embedding helpers + # ------------------------------------------------------------------ + + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor: + """Compatibility shim — unused at runtime (everything goes via inputs_embeds).""" + return self.text_embedding(input_ids) + + def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: + return self.get_input_embeddings(input_ids) + + def _embed_phoneme(self, phoneme_tokens: torch.Tensor) -> torch.Tensor: + """Average the per-stack phoneme embeddings (``[num_tokens, S] -> [num_tokens, dim]``).""" + acc = self.phoneme_embeddings[0](phoneme_tokens[:, 0]) + for s in range(1, len(self.phoneme_embeddings)): + acc = acc + self.phoneme_embeddings[s](phoneme_tokens[:, s]) + return acc / len(self.phoneme_embeddings) + + # ------------------------------------------------------------------ + # Decode-token dispatch (which positions need the local transformer) + # ------------------------------------------------------------------ + + def _get_decode_idxs(self): + """Return ``(decode_token_indices, num_requests)`` for code-predictor dispatch. + + Mirrors the qwen3-tts / eartts pattern: + + * ``(None, 0)`` → run the local transformer on every token (profile / + dummy run with no ``attn_metadata``, or a decode-only batch where + ``max_query_len == 1``), so the captured CUDA graph covers every + ``cudagraph_capture_sizes`` value. + * ``(indices, num_requests)`` → run only on the listed decode positions + (mixed prefill+decode batch). ``indices`` is padded to the next + captured graph size; ``num_requests`` is the unpadded count. + """ + ctx = get_forward_context() + attn_metadata = ctx.attn_metadata + if attn_metadata is None: + return None, 0 + + if isinstance(attn_metadata, dict): + any_layer_meta = next(iter(attn_metadata.values())) + else: + any_layer_meta = attn_metadata + + if any_layer_meta.max_query_len == 1: + return None, 0 + + start_loc = any_layer_meta.query_start_loc + tokens_per_req = start_loc[1:] - start_loc[:-1] + is_decode = tokens_per_req == 1 + decode_token_indices = start_loc[:-1][is_decode] + + num_requests = decode_token_indices.shape[0] + padded_num_requests = num_requests + if self.vllm_config.compilation_config.cudagraph_mode != CUDAGraphMode.NONE: + sizes = self.vllm_config.compilation_config.cudagraph_capture_sizes + idx = bisect.bisect_left(sizes, num_requests) + if idx < len(sizes): + padded_num_requests = sizes[idx] + if padded_num_requests != num_requests: + decode_token_indices = torch.nn.functional.pad( + decode_token_indices, (0, padded_num_requests - num_requests) + ) + return decode_token_indices, num_requests + + # ------------------------------------------------------------------ + # forward + # ------------------------------------------------------------------ + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + intermediate_tensors: Optional[IntermediateTensors] = None, + inputs_embeds: Optional[torch.Tensor] = None, + **_: Any, + ) -> torch.Tensor: + """Assemble the per-token embedding, run the backbone, then the codes. + + ``inputs_embeds`` carries the prefill embedding span produced by + :meth:`preprocess` (zeros at decode positions). For decode positions we + assemble ``text_emb + phoneme_emb + audio_emb`` in-place from the + per-token buffers, run the backbone, then sample the codebooks with the + local transformer (skipping prefill positions). + """ + num_tokens = input_ids.shape[0] + combined = self._combined_embeddings[:num_tokens] + if inputs_embeds is not None: + combined.copy_(inputs_embeds) + else: + combined.zero_() + + decode_idx, num_req = self._get_decode_idxs() + + if decode_idx is None: + # Profile / dummy run or decode-only batch: assemble decode + # embeddings everywhere so the captured graph sees the full path. + self._assemble_decode_embeddings(combined, slice(0, num_tokens)) + elif num_req > 0: + valid = decode_idx[:num_req] + self._assemble_decode_embeddings(combined, valid) + + hidden_states = self.backbone( + input_ids, + positions, + intermediate_tensors, + inputs_embeds=combined, + ) + + # Sample codes (local transformer) only where needed. + if decode_idx is None: + codes = self.code_predictor.generate_codes(hidden_states) + self._out_codes[:num_tokens].copy_(codes) + if self.has_phoneme: + self._predict_phonemes(hidden_states, slice(0, num_tokens)) + elif num_req > 0: + ctx = get_forward_context() + orig_bd = ctx.batch_descriptor + ctx.batch_descriptor = BatchDescriptor(num_tokens=decode_idx.shape[0]) + codes = self.code_predictor.generate_codes(hidden_states[decode_idx]) + ctx.batch_descriptor = orig_bd + valid = decode_idx[:num_req] + self._out_codes[valid] = codes[:num_req] + if self.has_phoneme: + self._predict_phonemes(hidden_states, valid) + + return hidden_states + + def _assemble_decode_embeddings(self, combined: torch.Tensor, idx) -> None: + """Add ``text + phoneme + audio`` embeddings into ``combined`` at ``idx``.""" + # Audio: previous-frame codes (gated by validity). + audio_codes = self._dec_audio_codes[idx] + audio_emb = self.code_predictor.embed_audio_frame(audio_codes) + audio_emb = audio_emb * self._dec_audio_valid[idx].unsqueeze(-1).to(audio_emb.dtype) + combined[idx] += audio_emb + + # Text: current subword token (gated by validity). + text_emb = self.text_embedding(self._dec_text_tokens[idx]) + text_emb = text_emb * self._dec_text_mask[idx].unsqueeze(-1).to(text_emb.dtype) + combined[idx] += text_emb + + # Phoneme: previous predicted phoneme (gated by validity). + if self.has_phoneme: + phon_emb = self._embed_phoneme(self._dec_phoneme_tokens[idx]) + phon_emb = phon_emb * self._dec_phoneme_valid[idx].unsqueeze(-1).to(phon_emb.dtype) + combined[idx] += phon_emb + + @torch.no_grad() + def _predict_phonemes(self, hidden_states: torch.Tensor, idx) -> None: + """Argmax the phoneme head and stash the prediction for the next step.""" + logits = self.phoneme_final_proj(hidden_states[idx].float()) + s = self.arch.phoneme_stacking_factor + logits = logits.view(-1, s, self.arch.phoneme_vocab_size) + self._dec_phoneme_tokens[idx] = logits.argmax(dim=-1).long() + self._dec_phoneme_valid[idx] = 1 + + # ------------------------------------------------------------------ + # compute_logits — dummy (real output is the codes tensor) + # ------------------------------------------------------------------ + + def compute_logits(self, hidden_states, sampling_metadata: Any = None) -> Optional[torch.Tensor]: + """Return zero logits so vLLM's sampler always picks index 0. + + The width is taken from ``hf_config.vocab_size`` so the sampler's + working buffers match. The sampled id is irrelevant — audio is surfaced + via :meth:`make_omni_output`. + """ + if isinstance(hidden_states, OmniOutput): + hidden_states = hidden_states.text_hidden_states + if hidden_states is None: + return None + batch_size = hidden_states.shape[0] + return hidden_states.new_zeros(batch_size, int(self.hf_config.vocab_size)) + + # ------------------------------------------------------------------ + # multimodal output plumbing + # ------------------------------------------------------------------ + + def make_omni_output(self, model_outputs, **_: Any) -> OmniOutput: + """Surface the sampled codes (``BT x num_codebooks``) under ``audio_codes``.""" + if isinstance(model_outputs, OmniOutput): + return model_outputs + hidden = model_outputs + num_tokens = int(hidden.shape[0]) + audio_codes = self._out_codes[:num_tokens].clone() + return OmniOutput( + text_hidden_states=hidden, + multimodal_outputs={"audio_codes": audio_codes}, + ) + + # ------------------------------------------------------------------ + # preprocess / postprocess + # ------------------------------------------------------------------ + + @staticmethod + def _unwrap(value: Any) -> Any: + if isinstance(value, list): + return value[0] if value else None + return value + + def preprocess( + self, + input_ids: torch.Tensor, + input_embeds: Optional[torch.Tensor], + *, + start: int = 0, + end: int = 0, + **info_dict: Any, + ) -> tuple[torch.Tensor, torch.Tensor, dict[str, Any]]: + """Build per-request ``(input_ids, inputs_embeds)`` for this step. + + Prefill (``span_len > 1``): slice the precomputed ``prompt_embeds`` + context embedding into this chunk and return it; ``input_ids`` are + placeholders. Decode (``span_len == 1``): write the per-token decode + inputs (previous codes, current text token, previous phoneme) into the + model buffers at ``start`` and return a zero embedding that + :meth:`forward` accumulates into. + """ + nested = info_dict.get("additional_information") + if isinstance(nested, dict): + merged = {k: v for k, v in info_dict.items() if k != "additional_information"} + for k, v in nested.items(): + merged.setdefault(k, v) + info_dict = merged + + device = input_ids.device + span_len = int(input_ids.shape[0]) + if span_len <= 0: + base = input_embeds if input_embeds is not None else self.embed_input_ids(input_ids) + return input_ids, base, {} + + if span_len > 1: + return self._preprocess_prefill(input_ids, span_len, device, info_dict) + return self._preprocess_decode(input_ids, start, device, info_dict) + + def _preprocess_prefill( + self, + input_ids: torch.Tensor, + span_len: int, + device: torch.device, + info_dict: dict[str, Any], + ) -> tuple[torch.Tensor, torch.Tensor, dict[str, Any]]: + prompt_embeds = self._unwrap(info_dict.get("prompt_embeds")) + if not isinstance(prompt_embeds, torch.Tensor) or prompt_embeds.ndim != 2: + raise ValueError( + "EasyMagpieTTS preprocess requires additional_information.prompt_embeds " + "of shape (T_ctx, embedding_dim) for prefill." + ) + prompt_embeds = prompt_embeds.to(device=device, dtype=self._combined_embeddings.dtype) + + offset = int(info_dict.get("ear_prefill_offset", 0) or 0) + total = int(prompt_embeds.shape[0]) + s = max(0, min(offset, total)) + e = max(0, min(offset + span_len, total)) + take = prompt_embeds[s:e] + if int(take.shape[0]) < span_len: + pad_n = span_len - int(take.shape[0]) + pad_rows = ( + take[-1:].expand(pad_n, -1) + if take.shape[0] > 0 + else prompt_embeds.new_zeros(pad_n, prompt_embeds.shape[-1]) + ) + take = torch.cat([take, pad_rows], dim=0) + + info_update = { + "ear_prefill_offset": offset + span_len, + "ear_decode_offset": 0, + } + input_ids_out = torch.full_like(input_ids, _DUMMY_TOKEN_ID) + return input_ids_out, take, info_update + + def _preprocess_decode( + self, + input_ids: torch.Tensor, + start: int, + device: torch.device, + info_dict: dict[str, Any], + ) -> tuple[torch.Tensor, torch.Tensor, dict[str, Any]]: + decode_offset = int(info_dict.get("ear_decode_offset", 0) or 0) + + # Text channel (streaming list that grows by one per step). + text_tokens = info_dict.get("text_tokens") + if isinstance(text_tokens, list) and text_tokens: + idx = min(decode_offset, len(text_tokens) - 1) + self._dec_text_tokens[start] = int(text_tokens[idx]) + self._dec_text_mask[start] = 1 + else: + self._dec_text_mask[start] = 0 + + # Phoneme channel: previous-step prediction stashed by postprocess. + if self.has_phoneme: + last_phon = info_dict.get("last_phoneme_token") + if isinstance(last_phon, torch.Tensor) and last_phon.numel() > 0: + p = last_phon.to(device=device, dtype=torch.long).reshape(-1) + self._dec_phoneme_tokens[start, : p.shape[0]].copy_(p[: self.arch.phoneme_stacking_factor]) + self._dec_phoneme_valid[start] = 1 + else: + self._dec_phoneme_valid[start] = 0 + + # Audio channel: previous-frame codes (BOS seed on the first step). + last_codes = info_dict.get("last_audio_codes") + if isinstance(last_codes, torch.Tensor) and last_codes.numel() > 0: + c = last_codes.to(device=device, dtype=torch.long).reshape(-1)[: self.num_codebooks] + self._dec_audio_codes[start, : c.shape[0]].copy_(c) + self._dec_audio_valid[start] = 1 + else: + # First decode step after prefill: seed with audio BOS. + self._dec_audio_codes[start].fill_(self.arch.audio_bos_id) + self._dec_audio_valid[start] = 1 + + inputs_embeds_out = torch.zeros((1, self.embedding_dim), device=device, dtype=self._combined_embeddings.dtype) + info_update = {"ear_decode_offset": decode_offset + 1} + return input_ids, inputs_embeds_out, info_update + + def postprocess(self, hidden_states: torch.Tensor, multimodal_outputs: Optional[dict[str, Any]] = None, **_: Any): + """Stash the last frame's codes (and phoneme) for the next decode step.""" + if hidden_states.numel() == 0: + return {} + stride0 = hidden_states.stride(0) or 1 + req_start = hidden_states.storage_offset() // stride0 + last = req_start + hidden_states.shape[0] - 1 + + out: dict[str, Any] = {} + audio_codes = (multimodal_outputs or {}).get("audio_codes") + if isinstance(audio_codes, torch.Tensor) and audio_codes.numel() > 0: + out["last_audio_codes"] = audio_codes[last : last + 1].detach() + if self.has_phoneme: + out["last_phoneme_token"] = self._dec_phoneme_tokens[last : last + 1].detach().clone() + return out + + # ------------------------------------------------------------------ + # weight loading + # ------------------------------------------------------------------ + + # Checkpoint prefixes (reference EasyMagpieTTS state dict) → in-model paths. + # ``decoder.*`` is fed to the vLLM backbone loader separately (it understands + # HF Qwen2 naming + qkv packing). The TTS submodules are copied manually. + _TTS_PREFIX_MAP = { + "local_transformer.": "code_predictor.local_transformer.", + "local_transformer_in_projection.": "code_predictor.local_transformer_in_projection.", + "local_transformer_audio_out_projection.": "code_predictor.local_transformer_audio_out_projection.", + "local_transformer_out_projections.": "code_predictor.local_transformer_out_projections.", + "audio_embeddings.": "code_predictor.audio_embeddings.", + "audio_in_projection.": "code_predictor.audio_in_projection.", + "phoneme_embeddings.": "phoneme_embeddings.", + "phoneme_final_proj.": "phoneme_final_proj.", + "text_embedding.": "text_embedding.", + } + + def _remap_tts_key(self, name: str) -> Optional[str]: + """Map a raw checkpoint key to its in-model parameter path (or ``None``).""" + for src, dst in self._TTS_PREFIX_MAP.items(): + if name.startswith(src): + return dst + name[len(src) :] + return None + + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + """Load backbone (Qwen2) + TTS submodule weights from a converted checkpoint. + + The converted checkpoint is expected to use the reference EasyMagpieTTS + key layout: the backbone under ``decoder.*`` (HF Qwen2 names) and the + TTS submodules at top level (``audio_embeddings.*``, ``local_transformer.*``, + ``phoneme_*``, ``text_embedding.*``, projection heads). Backbone weights + are routed to :meth:`Qwen2Model.load_weights` (which packs qkv / gate-up + and handles HF naming); TTS weights are copied directly by name. + """ + own_params = dict(self.named_parameters()) + loaded: set[str] = set() + backbone_weights: list[tuple[str, torch.Tensor]] = [] + + for name, tensor in weights: + if name.startswith("decoder."): + backbone_weights.append((name[len("decoder.") :], tensor)) + continue + mapped = self._remap_tts_key(name) + if mapped is None: + # Unrelated checkpoint section (codec, speaker encoder, CAS, etc.). + continue + target = own_params.get(mapped) + if target is None: + logger.warning("EasyMagpieTTS: no parameter for checkpoint key %s -> %s", name, mapped) + continue + if target.shape != tensor.shape: + raise RuntimeError( + f"EasyMagpieTTS weight shape mismatch at {mapped!r}: " + f"ckpt {tuple(tensor.shape)} vs model {tuple(target.shape)}" + ) + with torch.no_grad(): + target.data.copy_(tensor.to(target.dtype)) + loaded.add(mapped) + + backbone_loaded = self.backbone.load_weights(backbone_weights) + loaded |= {f"backbone.{n}" for n in backbone_loaded} + + # Derived runtime state. + self.code_predictor.init_forbidden_mask() + + # The backbone's vestigial embed_tokens table is never consumed + # (everything goes through inputs_embeds); don't flag it as missing. + loaded.add("backbone.embed_tokens.weight") + + logger.info("Loaded %d weights for EasyMagpieTTSForConditionalGeneration", len(loaded)) + return loaded diff --git a/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/local_transformer.py b/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/local_transformer.py new file mode 100644 index 000000000000..a72ee6ecd52d --- /dev/null +++ b/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/local_transformer.py @@ -0,0 +1,398 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""From-scratch autoregressive local transformer for EasyMagpieTTS on vLLM-Omni. + +The reference EasyMagpieTTS model predicts the ``C * S`` stacked audio +codebooks of one frame *autoregressively* with a small causal transformer +(``nemo.collections.tts.modules.transformer_2501.Transformer``) conditioned on +the backbone's per-frame hidden state. The reference implementation re-creates +fresh tensors and (optionally) a KV cache on every codebook step, which is +incompatible with CUDA-graph replay. + +This module re-implements that local transformer from scratch so it can run as +a single compiled CUDA graph: + +* :class:`EasyMagpieLocalTransformer` mirrors the ``transformer_2501`` + layer/weight layout **exactly** (so a stock checkpoint loads 1:1) but uses + ``scaled_dot_product_attention`` and drops the KV cache / padding-mask + plumbing. It is decorated with ``@support_torch_compile`` so vLLM captures + one CUDA graph for the fixed ``(num_tokens, num_stacked_codebooks, hidden)`` + input shape. +* :class:`EasyMagpieCodePredictor` owns the persistent, address-stable scratch + buffers and runs the per-frame autoregressive loop, re-invoking the compiled + transformer once per codebook over the **same** buffer (matching the + qwen3-tts code-predictor trick: replaying one fixed-shape graph N times is + faster and simpler than capturing N separate graphs). + +All sampling is CUDA-graph safe (Gumbel-max + ``topk`` + ``masked_fill`` only; +no host syncs, no ``multinomial`` on possibly-degenerate warmup data). +""" +from __future__ import annotations + +import torch +from torch import nn +from vllm.compilation.decorators import support_torch_compile +from vllm.config import VllmConfig + +from easymagpie_vllm_omni.config import EasyMagpieOmniArch + + +def _gumbel_argmax(logits: torch.Tensor) -> torch.Tensor: + """Gumbel-max categorical draw — CUDA-graph safe. + + Equivalent to sampling from ``softmax(logits)`` but uses only + ``uniform_`` + ``log`` + ``argmax`` (all legal inside a captured graph) + and degrades gracefully on degenerate warmup logits instead of triggering + a device-side assert the way ``multinomial`` does. + """ + u = torch.empty_like(logits).uniform_(1e-20, 1.0 - 1e-20) + return (logits - torch.log(-torch.log(u))).argmax(dim=-1) + + +def sample_codebook( + logits: torch.Tensor, + *, + temperature: float, + top_k: int, + forbidden_mask: torch.Tensor | None, +) -> torch.Tensor: + """Sample one codebook's tokens from logits (CUDA-graph safe). + + Args: + logits: ``[num_tokens, vocab]`` raw codebook logits. + temperature: Sampling temperature; ``<= 0`` falls back to argmax. + top_k: Top-k truncation width (``<= 0`` disables truncation). + forbidden_mask: Optional ``[vocab]`` bool mask; ``True`` entries are + set to ``-inf`` before sampling (reserved/special tokens). + + Returns: + ``[num_tokens]`` int64 sampled token ids. + """ + if forbidden_mask is not None: + logits = logits.masked_fill(forbidden_mask, float("-inf")) + + if temperature <= 0.0: + return logits.argmax(dim=-1) + + logits = logits / temperature + + if top_k is not None and top_k > 0: + vals, idxs = torch.topk(logits, k=min(top_k, logits.size(-1)), dim=-1) + sampled_in_k = _gumbel_argmax(vals) + return idxs.gather(-1, sampled_in_k.unsqueeze(-1)).squeeze(-1) + + return _gumbel_argmax(logits) + + +class EasyMagpieLTSelfAttention(nn.Module): + """Causal self-attention matching ``transformer_2501.SelfAttention`` weights. + + Same projections (``qkv_net`` fused QKV without bias, ``o_net`` without + bias) and the same ``d_head ** -0.5`` scaling, but computed with + ``scaled_dot_product_attention`` and an ``is_causal=True`` flag instead of + the materialised causal-mask buffer + naive softmax. No KV cache: the + autoregressive loop re-runs the full (short, fixed-length) sequence each + step, which is what makes the whole thing CUDA-graph capturable. + """ + + def __init__(self, d_model: int, n_heads: int) -> None: + super().__init__() + assert d_model % n_heads == 0, "d_model must be divisible by n_heads" + self.n_heads = n_heads + self.d_head = d_model // n_heads + self.scale = self.d_head**-0.5 + self.qkv_net = nn.Linear(d_model, 3 * n_heads * self.d_head, bias=False) + self.o_net = nn.Linear(n_heads * self.d_head, d_model, bias=False) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + b, t, _ = x.shape + qkv = self.qkv_net(x).reshape(b, t, 3, self.n_heads, self.d_head) + q, k, v = qkv.unbind(dim=2) # each [b, t, nh, dh] + # [b, nh, t, dh] + q = q.transpose(1, 2) + k = k.transpose(1, 2) + v = v.transpose(1, 2) + attn = torch.nn.functional.scaled_dot_product_attention( + q, k, v, is_causal=True, scale=self.scale + ) + attn = attn.transpose(1, 2).contiguous().view(b, t, -1) + return self.o_net(attn) + + +class EasyMagpieLTFeedForward(nn.Module): + """Positionwise FFN matching ``transformer_2501.PositionwiseConvFF`` weights. + + The reference uses ``Conv1d(kernel_size=1)`` layers named ``proj.conv`` and + ``o_net.conv`` (no bias). A kernel-1 conv is a plain linear over the channel + dim, so we keep the exact ``Conv1d`` submodule names — the checkpoint loads + 1:1 — and apply them with a single transpose, GELU(tanh) in between. + """ + + def __init__(self, d_model: int, d_ffn: int) -> None: + super().__init__() + # Wrap the Conv1d in a tiny container so the parameter path is + # ``proj.conv.weight`` / ``o_net.conv.weight`` exactly as in the + # reference ``ConvolutionLayer``. + self.proj = _Conv1dWrapper(d_model, d_ffn) + self.o_net = _Conv1dWrapper(d_ffn, d_model) + self.act = nn.GELU(approximate="tanh") + + def forward(self, x: torch.Tensor) -> torch.Tensor: + # x: [b, t, c] -> conv expects [b, c, t] + h = x.transpose(1, 2) + h = self.act(self.proj(h)) + h = self.o_net(h) + return h.transpose(1, 2) + + +class _Conv1dWrapper(nn.Module): + """Holds a kernel-1 ``Conv1d`` under attribute name ``conv`` (no bias).""" + + def __init__(self, in_ch: int, out_ch: int) -> None: + super().__init__() + self.conv = nn.Conv1d(in_ch, out_ch, kernel_size=1, bias=False) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.conv(x) + + +class EasyMagpieLTLayer(nn.Module): + """One pre-norm transformer layer (self-attn + FFN), bias-free LayerNorms. + + Residual structure matches ``transformer_2501.TransformerLayer`` with an + all-ones ``x_mask`` (inference): ``x = x + attn(norm_self(x))`` then + ``x = x + ff(norm_pos_ff(x))``. The ``x * x_mask`` multiplications are + identities when nothing is padded, so they are dropped. + """ + + def __init__(self, d_model: int, d_ffn: int, n_heads: int) -> None: + super().__init__() + self.norm_self = nn.LayerNorm(d_model, bias=False) + self.self_attention = EasyMagpieLTSelfAttention(d_model, n_heads) + self.norm_pos_ff = nn.LayerNorm(d_model, bias=False) + self.pos_ff = EasyMagpieLTFeedForward(d_model, d_ffn) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = x + self.self_attention(self.norm_self(x)) + x = x + self.pos_ff(self.norm_pos_ff(x)) + return x + + +# NOTE: ``dynamic_arg_dims`` is passed explicitly rather than relying on +# vLLM's annotation-based inference. This file uses +# ``from __future__ import annotations`` (PEP 563), so ``forward``'s +# annotations are stored as strings (``"torch.Tensor"``) and vLLM's +# ``v.annotation in [torch.Tensor, ...]`` check would never match, raising +# "No dynamic dimensions found...". ``inputs_embeds`` is +# ``[num_tokens, num_codebooks, hidden]`` -> dim 0 (num_tokens) is dynamic. +@support_torch_compile(dynamic_arg_dims={"inputs_embeds": 0}) +class EasyMagpieLocalTransformer(nn.Module): + """Compiled causal transformer stack with learnable positional embeddings. + + Decorated with ``@support_torch_compile`` so vLLM captures a single CUDA + graph for the fixed ``(num_tokens, num_stacked_codebooks, d_model)`` input + shape. Weight layout mirrors ``transformer_2501.Transformer``: + ``position_embeddings`` (learnable), ``layers.{i}.*`` and a no-op + ``norm_out`` (``apply_norm_out=False`` in the reference, hence ``Identity``). + """ + + def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: + super().__init__() + arch = EasyMagpieOmniArch.from_hf_config(vllm_config.model_config.hf_config) + d_model = arch.local_transformer_hidden_dim + n_heads = arch.local_transformer_n_heads + n_layers = arch.local_transformer_n_layers + d_ffn = d_model * 4 + # +2 matches the reference ``max_length_causal_mask`` head-room + # (``num_stacked_codebooks + 2``). + max_len = arch.num_stacked_codebooks + 2 + + self.position_embeddings = nn.Embedding(max_len, d_model) + self.layers = nn.ModuleList( + [EasyMagpieLTLayer(d_model, d_ffn, n_heads) for _ in range(n_layers)] + ) + # apply_norm_out=False in the reference config -> no parameters. + self.norm_out = nn.Identity() + + def forward(self, inputs_embeds: torch.Tensor) -> torch.Tensor: + seq_len = inputs_embeds.shape[1] + positions = torch.arange(seq_len, device=inputs_embeds.device) + x = inputs_embeds + self.position_embeddings(positions).unsqueeze(0) + for layer in self.layers: + x = layer(x) + return self.norm_out(x) + + +class EasyMagpieCodePredictor(nn.Module): + """Autoregressive intra-frame codebook predictor (the "local transformer"). + + Given the backbone's per-frame hidden state, predicts all ``C * S`` stacked + audio codebooks one at a time. Owns the codebook input embeddings (shared + with the outer model for building decode-step input embeddings) and all the + projection heads, plus the persistent scratch buffers required for + CUDA-graph replay. + + Per frame (``generate_codes``): + + 1. Position 0 of the input buffer holds ``in_proj(dec_hidden)``. + 2. For codebook ``k`` in ``0 .. N-1``: run the compiled transformer over the + whole buffer, read row ``k`` of the output, project to codebook-``k`` + logits, sample, and (if ``k < N-1``) write ``in_proj(audio_emb_k(code))`` + into buffer row ``k + 1``. + + The buffer is zeroed once per frame and filled incrementally; because the + transformer is causal, rows ``> k`` never influence row ``k``, so replaying + the same fixed-shape graph N times yields the correct autoregressive result. + """ + + def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: + super().__init__() + arch = EasyMagpieOmniArch.from_hf_config(vllm_config.model_config.hf_config) + self.arch = arch + self.num_codebooks = arch.num_stacked_codebooks + self.num_tokens_per_codebook = arch.num_all_tokens_per_codebook + self.audio_embedding_dim = arch.audio_embedding_dim + self.embedding_dim = arch.embedding_dim + lt_hidden = arch.local_transformer_hidden_dim + + # Per-codebook audio token embeddings (shared with the outer model's + # decode-step input-embedding assembly). Names match the reference + # checkpoint's ``audio_embeddings.{i}``. + self.audio_embeddings = nn.ModuleList( + [nn.Embedding(self.num_tokens_per_codebook, self.audio_embedding_dim) for _ in range(self.num_codebooks)] + ) + # audio_embedding_dim -> embedding_dim (Identity when equal). + if self.audio_embedding_dim != self.embedding_dim: + self.audio_in_projection = nn.Linear(self.audio_embedding_dim, self.embedding_dim) + else: + self.audio_in_projection = nn.Identity() + + # embedding_dim (== backbone hidden) -> local-transformer hidden. + if lt_hidden != self.embedding_dim: + self.local_transformer_in_projection = nn.Linear(self.embedding_dim, lt_hidden) + else: + self.local_transformer_in_projection = nn.Identity() + + self.local_transformer = EasyMagpieLocalTransformer( + vllm_config=vllm_config, prefix=f"{prefix}.local_transformer" + ) + + # local-transformer hidden -> audio_embedding_dim (Identity when equal). + if self.audio_embedding_dim != lt_hidden: + self.local_transformer_audio_out_projection = nn.Linear(lt_hidden, self.audio_embedding_dim) + else: + self.local_transformer_audio_out_projection = nn.Identity() + + # Per-codebook output heads. + self.local_transformer_out_projections = nn.ModuleList( + [nn.Linear(self.audio_embedding_dim, self.num_tokens_per_codebook) for _ in range(self.num_codebooks)] + ) + + # Forbidden-token mask (reserved/special tokens, EOS kept reachable). + # Populated by :meth:`init_forbidden_mask` once arch ids are known. + self.register_buffer( + "forbidden_mask", + torch.zeros(self.num_tokens_per_codebook, dtype=torch.bool), + persistent=False, + ) + + # Sampling knobs (overridable from the outer model / request). + self.temperature: float = 0.7 + self.top_k: int = 80 + + # ── Persistent address-stable scratch buffers ────────────────── + max_num_tokens = vllm_config.scheduler_config.max_num_batched_tokens + dtype = vllm_config.model_config.dtype + self._buf_inputs = torch.zeros(max_num_tokens, self.num_codebooks, lt_hidden, dtype=dtype) + self._out_codes = torch.zeros(max_num_tokens, self.num_codebooks, dtype=torch.long) + + @torch.no_grad() + def init_forbidden_mask(self) -> None: + """Forbid all trailing special tokens except audio EOS. + + Mirrors ``SpecialAudioToken.get_forbidden_tokens`` — everything in the + special-token block above ``codebook_size`` is blocked at sampling + time, except ``audio_eos`` which must remain reachable to terminate. + """ + mask = torch.zeros(self.num_tokens_per_codebook, dtype=torch.bool, device=self.forbidden_mask.device) + mask[self.arch.codebook_size :] = True + eos = self.arch.audio_eos_id + if 0 <= eos < self.num_tokens_per_codebook: + mask[eos] = False + self.forbidden_mask.copy_(mask) + + def embed_codebook(self, codebook_idx: int, codes: torch.Tensor) -> torch.Tensor: + """Embed a single codebook's tokens (``[num_tokens] -> [num_tokens, audio_dim]``).""" + return self.audio_embeddings[codebook_idx](codes) + + def embed_audio_frame(self, codes: torch.Tensor) -> torch.Tensor: + """Embed a full frame of stacked codes into the backbone embedding space. + + Averages per-codebook embeddings then applies ``audio_in_projection``, + matching the reference ``embed_audio_tokens`` (which sums and divides by + the number of codebooks). Used by the outer model to build the decode + input embedding from the previous frame's codes. + + Args: + codes: ``[num_tokens, num_codebooks]`` int64 codes. + + Returns: + ``[num_tokens, embedding_dim]`` float embedding. + """ + acc = self.audio_embeddings[0](codes[:, 0]) + for c in range(1, self.num_codebooks): + acc = acc + self.audio_embeddings[c](codes[:, c]) + acc = acc / self.num_codebooks + return self.audio_in_projection(acc) + + def forward(self, inputs_embeds: torch.Tensor) -> torch.Tensor: + """Run the compiled local transformer over the input buffer.""" + return self.local_transformer(inputs_embeds) + + @torch.no_grad() + def generate_codes(self, dec_hidden: torch.Tensor) -> torch.Tensor: + """Autoregressively sample all ``C * S`` codebooks for each frame. + + Args: + dec_hidden: ``[num_tokens, hidden]`` backbone hidden state (one row + per frame being decoded). + + Returns: + ``[num_tokens, num_codebooks]`` int64 sampled codes. + """ + num_tokens = dec_hidden.shape[0] + buf = self._buf_inputs[:num_tokens] + out = self._out_codes[:num_tokens] + buf.zero_() + + # Row 0: projected backbone hidden state (the AR "prompt"). + buf[:, 0, :] = self.local_transformer_in_projection(dec_hidden) + + forbidden = self.forbidden_mask if self.forbidden_mask.any() else None + for k in range(self.num_codebooks): + hidden = self(buf) # compiled transformer over the fixed buffer + row = self.local_transformer_audio_out_projection(hidden[:, k, :]) + logits = self.local_transformer_out_projections[k](row) + code_k = sample_codebook( + logits, + temperature=self.temperature, + top_k=self.top_k, + forbidden_mask=forbidden, + ) + out[:, k] = code_k + if k + 1 < self.num_codebooks: + emb = self.audio_in_projection(self.audio_embeddings[k](code_k)) + buf[:, k + 1, :] = self.local_transformer_in_projection(emb) + + return out[:num_tokens] diff --git a/examples/tts/easymagpie_vllm_omni/pyproject.toml b/examples/tts/easymagpie_vllm_omni/pyproject.toml new file mode 100644 index 000000000000..5cd41cead748 --- /dev/null +++ b/examples/tts/easymagpie_vllm_omni/pyproject.toml @@ -0,0 +1,20 @@ +[build-system] +requires = ["setuptools>=64"] +build-backend = "setuptools.build_meta" + +[project] +name = "easymagpie-vllm-omni" +version = "0.1.0" +description = "vLLM-Omni model definition for EasyMagpieTTS (Qwen2 backbone + AR local transformer)" +requires-python = ">=3.10" +dependencies = [ + # The heavy runtime deps (vllm, vllm-omni, torch) are provided by the + # target vllm_omni_env. Treat this as "install into an already-bootstrapped + # vllm_omni_env"; do not install into NeMo's nemo_virtual_environment. +] + +[project.entry-points."vllm.general_plugins"] +easymagpie_omni = "vllm_plugin_easymagpie_omni:register" + +[tool.setuptools.packages.find] +include = ["easymagpie_vllm_omni*", "vllm_plugin_easymagpie_omni*"] diff --git a/examples/tts/easymagpie_vllm_omni/vllm_plugin_easymagpie_omni/__init__.py b/examples/tts/easymagpie_vllm_omni/vllm_plugin_easymagpie_omni/__init__.py new file mode 100644 index 000000000000..a050ed562788 --- /dev/null +++ b/examples/tts/easymagpie_vllm_omni/vllm_plugin_easymagpie_omni/__init__.py @@ -0,0 +1,51 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""vLLM plugin: register ``EasyMagpieTTS`` as a model architecture for vLLM-Omni. + +Loaded by vLLM in the parent process and each EngineCore subprocess via the +``vllm.general_plugins`` entry point. The lazy ``:`` target means +the (NeMo-free) model module is only imported when vLLM resolves the +architecture, keeping heavy imports out of the parent process. +""" + +_TARGET = "easymagpie_vllm_omni.easymagpie:EasyMagpieTTSForConditionalGeneration" +_ARCHS = ("EasyMagpieTTS", "EasyMagpieTTSForConditionalGeneration") + + +def register() -> None: + """Register the model class under all supported arch names. + + The architecture must be registered in **both** registries: + + * ``vllm.ModelRegistry`` — the stock vLLM global registry. + * ``vllm_omni``'s ``OmniModelRegistry`` — a *separate* ``_ModelRegistry`` + instance that the vLLM-Omni engine actually consults when resolving a + model architecture. Registering only in the stock registry leaves the + omni engine reporting ``Model architectures [...] are not supported``. + """ + from vllm import ModelRegistry + + registries = [ModelRegistry] + try: + from vllm_omni.model_executor.models import OmniModelRegistry + + registries.append(OmniModelRegistry) + except Exception: + # vllm_omni not installed — stock vLLM registration is enough. + pass + + for registry in registries: + for arch in _ARCHS: + if arch not in registry.get_supported_archs(): + registry.register_model(arch, _TARGET) From 87d742c859a20b2ac66bc53504c9bf9e119a5e2d Mon Sep 17 00:00:00 2001 From: Viacheslav Klimkov Date: Tue, 2 Jun 2026 18:20:44 +0200 Subject: [PATCH 03/45] examples/tts/easymagpie_vllm_omni: switch to actual configuration Signed-off-by: Viacheslav Klimkov --- .../easymagpie_inference_demo.ipynb | 115 ++++++++++---- .../easymagpie_vllm_omni/__init__.py | 11 +- .../easymagpie_vllm_omni/backbone_patches.py | 64 ++++++++ .../easymagpie_vllm_omni/config.py | 12 +- .../easymagpie_vllm_omni/easymagpie.py | 150 ++++++++++++------ .../tts/easymagpie_vllm_omni/pyproject.toml | 2 +- 6 files changed, 262 insertions(+), 92 deletions(-) create mode 100644 examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/backbone_patches.py diff --git a/examples/tts/easymagpie_vllm_omni/easymagpie_inference_demo.ipynb b/examples/tts/easymagpie_vllm_omni/easymagpie_inference_demo.ipynb index 0e2693776adf..88af6037e056 100644 --- a/examples/tts/easymagpie_vllm_omni/easymagpie_inference_demo.ipynb +++ b/examples/tts/easymagpie_vllm_omni/easymagpie_inference_demo.ipynb @@ -28,11 +28,12 @@ " engine accumulates them across steps just like eartts, so we trim to the last\n", " `len(token_ids)` decoded rows.\n", "\n", - "> **Dummy weights.** We build a tiny `config.json` (small backbone + small\n", - "> codebooks) and start the engine with `load_format=\\\"dummy\\\"`, so vLLM fills all\n", - "> parameters with random values. The emitted codes are therefore meaningless —\n", - "> this is a *smoke test* of the engine wiring, not a real synthesis. Point the\n", - "> engine at a real converted checkpoint (and drop `load_format`) to get audio.\n", + "> **Dummy weights.** We build a `config.json` sized to the real checkpoint\n", + "> (`2605_EMTTS_SmallMamba_Step150k_posttrained_epoch12.nemo`) and start the\n", + "> engine with `load_format=\\\"dummy\\\"`, so vLLM fills all parameters with random\n", + "> values. The emitted codes are therefore meaningless — this is a *smoke test*\n", + "> of the engine wiring, not a real synthesis. Point the engine at a real\n", + "> converted checkpoint (and drop `load_format`) to get audio.\n", "\n", "> **Environment.** Run this inside the bootstrapped `vllm_omni_env` (vLLM +\n", "> vLLM-Omni + compatible torch) with the plugin installed:\n", @@ -82,14 +83,24 @@ "## 1. Build a tiny dummy model directory\n", "\n", "The engine only needs a `config.json` that (a) names the registered arch and\n", - "(b) carries the EasyMagpie + Qwen2 scalars. We deliberately pick **small** dims\n", - "so the dummy backbone and local transformer are fast to instantiate.\n", + "(b) carries the EasyMagpie + Nemotron-H scalars. Here we size everything to match\n", + "the real checkpoint\n", + "`2605_NemotronTTS_V0.2/v2/2605_EMTTS_SmallMamba_Step150k_posttrained_epoch12.nemo`\n", + "(hidden 1536, 8 codebooks × 1024, frame-stacking ×2, 3-layer local transformer).\n", + "\n", + "The backbone is a **Nemotron-H** hybrid (Mamba2 + attention + MoE) decoder:\n", + "`EasyMagpieTTSForConditionalGeneration` constructs vLLM's `NemotronHModel` and\n", + "implements the hybrid-Mamba interfaces (`HasInnerState` / `IsHybrid` /\n", + "`SupportsMambaPrefixCaching`), exactly like the EasyMagpie vLLM *sidecar*. The\n", + "`nemotron_h_config` fields (`hybrid_override_pattern`, `mamba_*`, `n_routed_experts`,\n", + "…) are copied verbatim from the checkpoint.\n", "\n", "The EasyMagpie-specific scalars (`embedding_dim`, `num_audio_codebooks`,\n", "`codebook_size`, `frame_stacking_factor`, `local_transformer_*`, …) are read by\n", - "`EasyMagpieOmniArch.from_hf_config`; the standard Qwen2 fields (`hidden_size`,\n", - "`num_hidden_layers`, …) configure the reused `Qwen2Model` backbone. Setting\n", - "`phoneme_vocab_size = 0` disables the optional phoneme branch for simplicity.\n", + "`EasyMagpieOmniArch.from_hf_config`. The phoneme branch is **enabled**\n", + "(`phoneme_stacking_factor = 1`, `phoneme_vocab_size = 2051`) to match the\n", + "checkpoint; the model self-predicts phonemes, so no phoneme stream needs to be\n", + "supplied in the prompt.\n", "\n", "With `load_format=\\\"dummy\\\"` (set in the stage config) vLLM never reads weight\n", "files, so a lone `config.json` is enough — no safetensors, no tokenizer." @@ -102,32 +113,70 @@ "metadata": {}, "outputs": [], "source": [ - "# Small, internally-consistent dummy profile.\n", + "# Config matching the real checkpoint:\n", + "# 2605_NemotronTTS_V0.2/v2/2605_EMTTS_SmallMamba_Step150k_posttrained_epoch12.nemo\n", + "#\n", + "# The backbone is a Nemotron-H hybrid (Mamba2 + attention + MoE) decoder, wired\n", + "# through vLLM's `NemotronHModel` by `EasyMagpieTTSForConditionalGeneration`. The\n", + "# fields below are ported verbatim from the checkpoint's `model_config.yaml`\n", + "# (the `nemotron_h_config` block + the EasyMagpie scalars). With\n", + "# `load_format=\"dummy\"` the weights are random — a realistically-sized smoke test.\n", + "#\n", "# embedding_dim == hidden_size == audio_embedding_dim == local_transformer_hidden_dim\n", - "# keeps every in/out projection an Identity (fewer dummy params, same code path).\n", - "HIDDEN = 256\n", - "NUM_AUDIO_CODEBOOKS = 4\n", - "CODEBOOK_SIZE = 64\n", - "FRAME_STACKING = 2 # -> num_stacked_codebooks = NUM_AUDIO_CODEBOOKS * FRAME_STACKING = 8\n", - "TEXT_VOCAB = 256\n", + "# (all 1536 in the checkpoint) keeps every in/out projection an Identity.\n", + "HIDDEN = 1536 # nemotron_h_config.hidden_size / embedding_dim / audio_embedding_dim\n", + "NUM_AUDIO_CODEBOOKS = 8 # vector_quantizer.num_groups\n", + "CODEBOOK_SIZE = 1024 # prod(vector_quantizer.num_levels_per_group) = 4**5\n", + "FRAME_STACKING = 2 # -> num_stacked_codebooks = NUM_AUDIO_CODEBOOKS * FRAME_STACKING = 16\n", + "PHONEME_STACKING = 1 # phoneme_stacking_factor\n", + "PHONEME_VOCAB = 2051 # IPA-BPE 2048 tokenizer + 3 special tokens\n", + "TEXT_VOCAB = 131072 # nemotron_h_config.vocab_size\n", "\n", "config = {\n", " # Resolved through the `vllm.general_plugins` entry point registered by the\n", " # `easymagpie_vllm_omni` package -> EasyMagpieTTSForConditionalGeneration.\n", " \"architectures\": [\"EasyMagpieTTSForConditionalGeneration\"],\n", - " # Standard Qwen2 backbone fields (consumed by vllm Qwen2Model).\n", - " \"model_type\": \"qwen2\",\n", + " # Nemotron-H backbone fields (consumed by vllm NemotronHModel) — copied\n", + " # verbatim from the checkpoint's `nemotron_h_config` block.\n", + " \"model_type\": \"nemotron_h\",\n", " \"hidden_size\": HIDDEN,\n", - " \"intermediate_size\": 4 * HIDDEN,\n", - " \"num_hidden_layers\": 2,\n", - " \"num_attention_heads\": 4,\n", - " \"num_key_value_heads\": 4,\n", - " \"max_position_embeddings\": 4096,\n", - " \"rms_norm_eps\": 1e-6,\n", - " \"rope_theta\": 1000000.0,\n", + " \"num_hidden_layers\": 31,\n", " \"vocab_size\": TEXT_VOCAB,\n", + " \"num_attention_heads\": 12,\n", + " \"num_key_value_heads\": 4,\n", + " \"attention_dropout\": 0.0,\n", + " \"attention_bias\": False,\n", + " \"max_position_embeddings\": 8192,\n", + " \"mamba_num_heads\": 64,\n", + " \"mamba_head_dim\": 24,\n", + " \"ssm_state_size\": 128,\n", + " \"conv_kernel\": 4,\n", + " \"n_groups\": 8,\n", + " \"chunk_size\": 256,\n", + " \"mamba_hidden_act\": \"silu\",\n", + " \"use_conv_bias\": True,\n", + " \"use_bias\": False,\n", + " \"intermediate_size\": 4096,\n", + " \"mlp_hidden_act\": \"silu\",\n", + " \"mlp_bias\": False,\n", + " \"n_routed_experts\": 24,\n", + " \"num_experts_per_tok\": 4,\n", + " \"moe_intermediate_size\": 768,\n", + " \"moe_shared_expert_intermediate_size\": 2048,\n", + " \"n_group\": 1,\n", + " \"topk_group\": 1,\n", + " \"routed_scaling_factor\": 2.5,\n", + " \"norm_topk_prob\": True,\n", + " # 31-char layer pattern: M=Mamba2, *=attention, E=MLP/MoE (len == num_hidden_layers).\n", + " \"hybrid_override_pattern\": \"MEMEM*EMEMEM*EMEMEMEM*EMEMEMEME\",\n", + " \"layer_norm_epsilon\": 1e-5,\n", + " \"residual_in_fp32\": False,\n", " \"tie_word_embeddings\": False,\n", - " \"torch_dtype\": \"float32\",\n", + " # bfloat16, not float32: the Nemotron-H MoE layers run vLLM's fused-MoE\n", + " # Triton kernel, whose block sizes are tuned for 16-bit. In float32 the\n", + " # kernel needs ~2x shared memory and overflows the GPU limit\n", + " # (OutOfResources: shared memory). bf16 also matches the real checkpoint.\n", + " \"torch_dtype\": \"bfloat16\",\n", " # EasyMagpie-specific scalars (read by EasyMagpieOmniArch.from_hf_config).\n", " \"text_vocab_size\": TEXT_VOCAB,\n", " \"embedding_dim\": HIDDEN,\n", @@ -135,10 +184,10 @@ " \"num_audio_codebooks\": NUM_AUDIO_CODEBOOKS,\n", " \"codebook_size\": CODEBOOK_SIZE,\n", " \"frame_stacking_factor\": FRAME_STACKING,\n", - " \"phoneme_stacking_factor\": 0, # disable phoneme branch\n", - " \"phoneme_vocab_size\": 0,\n", - " \"local_transformer_n_layers\": 2,\n", - " \"local_transformer_n_heads\": 4,\n", + " \"phoneme_stacking_factor\": PHONEME_STACKING,\n", + " \"phoneme_vocab_size\": PHONEME_VOCAB,\n", + " \"local_transformer_n_layers\": 3,\n", + " \"local_transformer_n_heads\": 12,\n", " \"local_transformer_hidden_dim\": HIDDEN,\n", "}\n", "\n", @@ -220,7 +269,9 @@ " \"distributed_executor_backend\": \"uni\",\n", " \"max_num_batched_tokens\": MAX_NUM_BATCHED_TOKENS,\n", " \"max_model_len\": MAX_MODEL_LEN,\n", - " \"dtype\": \"float32\",\n", + " # bf16 (not fp32): the Nemotron-H fused-MoE Triton kernel's block\n", + " # sizes are tuned for 16-bit and overflow shared memory in fp32.\n", + " \"dtype\": \"bfloat16\",\n", " \"attention_backend\": \"TRITON_ATTN\",\n", " # --- dummy-weights smoke-test knobs ---\n", " \"load_format\": \"dummy\",\n", diff --git a/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/__init__.py b/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/__init__.py index 8a37af8454ea..074c48463276 100644 --- a/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/__init__.py +++ b/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/__init__.py @@ -14,14 +14,15 @@ """EasyMagpieTTS model definition for vLLM-Omni. This package provides an inference-only re-implementation of EasyMagpieTTS -(decoder-only, Qwen2 backbone + autoregressive local transformer over the -stacked audio codebooks) that plugs into the vLLM-Omni serving stack via the -standard ``preprocess`` / ``postprocess`` / ``make_omni_output`` hooks. +(decoder-only, Nemotron-H hybrid-Mamba backbone + autoregressive local +transformer over the stacked audio codebooks) that plugs into the vLLM-Omni +serving stack via the standard ``preprocess`` / ``postprocess`` / +``make_omni_output`` hooks. The companion ``vllm_plugin_easymagpie_omni`` package registers the model with vLLM's ``ModelRegistry`` through the ``vllm.general_plugins`` entry point. """ -from easymagpie_vllm_omni.config import EASYMAGPIE_QWEN, EasyMagpieOmniArch +from easymagpie_vllm_omni.config import EASYMAGPIE_SMALLMAMBA, EasyMagpieOmniArch -__all__ = ["EASYMAGPIE_QWEN", "EasyMagpieOmniArch"] +__all__ = ["EASYMAGPIE_SMALLMAMBA", "EasyMagpieOmniArch"] diff --git a/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/backbone_patches.py b/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/backbone_patches.py new file mode 100644 index 000000000000..efe8421f7af2 --- /dev/null +++ b/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/backbone_patches.py @@ -0,0 +1,64 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Backbone-side patches applied at model ``__init__``. + +Runtime fixes for the constructed ``NemotronHModel`` backbone. They live with +the model because they're inherent to running EasyMagpie SmallMamba +(``mlp_hidden_act=silu``) on vLLM's NemotronH implementation. Mirrors the +EasyMagpie vLLM *sidecar* (``easymagpie_vllm/backbone_patches.py``). +""" +from __future__ import annotations + +import torch.nn as nn +import torch.nn.functional as F +from vllm.logger import init_logger + +logger = init_logger(__name__) + + +class _SiluActivation(nn.Module): + """``nn.Module`` wrapper around ``F.silu`` (so vLLM's NemotronHMLP can hold it).""" + + def forward(self, x): + return F.silu(x) + + +def patch_silu_shared_experts(backbone) -> int: + """Replace ``shared_experts.act_fn`` with SiLU on every NemotronHMoE layer. + + vLLM's ``NemotronHMLP`` hard-codes ReLU² for ``shared_experts`` (ignoring + ``config.mlp_hidden_act``). SmallMamba trained with SiLU, so the mismatch + blows up shared-expert norms ~5× and the per-layer cosine drops to ≈-0.7 by + layer 30. Patching only ``act_fn`` (not the whole forward) keeps + ``NemotronHMLP.forward`` in charge so torch.compile / CUDA-graph capture + continue to wrap it unchanged. + + Args: + backbone: the ``NemotronHModel`` instance. + + Returns: + Number of layers patched. + """ + patched = 0 + for layer in backbone.layers: + mixer = getattr(layer, "mixer", None) + if mixer is None or mixer.__class__.__name__ != "NemotronHMoE": + continue + se = getattr(mixer, "shared_experts", None) + if se is None: + continue + se.act_fn = _SiluActivation() + patched += 1 + logger.info("SiLU shared_experts fix installed on %d layers", patched) + return patched diff --git a/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/config.py b/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/config.py index c569089e32f7..cdba51613a8c 100644 --- a/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/config.py +++ b/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/config.py @@ -13,9 +13,9 @@ # limitations under the License. """Architecture constants for the EasyMagpieTTS vLLM-Omni model. -These mirror the values baked into the reference EasyMagpieTTS checkpoint -(``examples/tts/conf/magpietts/easy_magpietts.yaml`` — Qwen2.5-1.5B backbone, -8 codebooks, frame-stacking ×2, 3-layer autoregressive local transformer). +These mirror the values baked into the reference EasyMagpieTTS SmallMamba +checkpoint (Nemotron-H hybrid Mamba2 + attention + MoE backbone, 8 codebooks, +frame-stacking ×2, 3-layer autoregressive local transformer). The vLLM-Omni model reads the bulk of its configuration from the ``hf_config`` provided by vLLM at construction time; this dataclass captures @@ -123,7 +123,7 @@ def from_hf_config(cls, hf_config: Any) -> "EasyMagpieOmniArch": Any attribute present on ``hf_config`` overrides the default profile; unknown attributes are ignored. This lets a converted checkpoint carry its own ``easymagpie`` block in ``config.json`` while still working - out-of-the-box on the reference Qwen2.5-1.5B profile. + out-of-the-box on the reference SmallMamba profile. """ defaults = cls() kwargs: dict[str, Any] = {} @@ -154,5 +154,5 @@ def from_hf_config(cls, hf_config: Any) -> "EasyMagpieOmniArch": return cls(**merged) -# Reference profile: Qwen2.5-1.5B backbone EasyMagpieTTS checkpoint. -EASYMAGPIE_QWEN = EasyMagpieOmniArch() +# Reference profile: Nemotron-H SmallMamba EasyMagpieTTS checkpoint. +EASYMAGPIE_SMALLMAMBA = EasyMagpieOmniArch() diff --git a/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/easymagpie.py b/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/easymagpie.py index bfb76ccb9303..e188b9387a7b 100644 --- a/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/easymagpie.py +++ b/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/easymagpie.py @@ -14,20 +14,27 @@ """Inference-only EasyMagpieTTS model for vLLM-Omni. EasyMagpieTTS is a decoder-only streaming TTS model: a text-LM backbone (the -reference checkpoint uses Qwen2.5-1.5B) consumes a per-frame additive input -embedding (text + phoneme + audio) and emits a per-frame hidden state, from -which a small autoregressive *local transformer* samples all ``C * S`` stacked -audio codebooks for that frame (see :mod:`easymagpie_vllm_omni.local_transformer`). +SmallMamba checkpoint uses a Nemotron-H hybrid Mamba2 + attention + MoE decoder) +consumes a per-frame additive input embedding (text + phoneme + audio) and +emits a per-frame hidden state, from which a small autoregressive *local +transformer* samples all ``C * S`` stacked audio codebooks for that frame +(see :mod:`easymagpie_vllm_omni.local_transformer`). This module wires that architecture into vLLM-Omni's ``preprocess`` / ``forward`` / ``compute_logits`` / ``make_omni_output`` / ``postprocess`` contract, following the same conventions as the upstream qwen3-tts and eartts vLLM-Omni model definitions: -* **Backbone** — vLLM's :class:`~vllm.model_executor.models.qwen2.Qwen2Model`, - reused wholesale (KV cache + paged attention) the same way the EasyMagpie - vLLM *sidecar* reuses ``NemotronHModel``. Every step feeds the backbone via - ``inputs_embeds``; its own ``embed_tokens`` table is never consumed. +* **Backbone** — vLLM's + :class:`~vllm.model_executor.models.nemotron_h.NemotronHModel`, reused + wholesale (hybrid Mamba2 state + KV cache + paged attention) exactly like the + EasyMagpie vLLM *sidecar*. Every step feeds the backbone via ``inputs_embeds``; + its own ``embed_tokens`` table is never consumed. Because the backbone is a + hybrid-Mamba model, the class implements vLLM's + :class:`HasInnerState` / :class:`IsHybrid` / :class:`SupportsMambaPrefixCaching` + contracts (mamba-state shape/dtype/copy helpers are delegated to + :class:`NemotronHForCausalLM`), and the SmallMamba SiLU shared-experts fix is + applied at construction (see :mod:`easymagpie_vllm_omni.backbone_patches`). * **Local transformer** — :class:`EasyMagpieCodePredictor`, a from-scratch, CUDA-graph-capturable re-implementation that runs as a single compiled graph. * **compute_logits** — returns trivial logits (à la eartts) so vLLM's sampler @@ -59,16 +66,21 @@ import torch from torch import nn from vllm.compilation.backends import set_model_tag -from vllm.compilation.decorators import ignore_torch_compile, support_torch_compile from vllm.config import CUDAGraphMode, VllmConfig from vllm.forward_context import BatchDescriptor, get_forward_context from vllm.logger import init_logger -from vllm.model_executor.models.qwen2 import Qwen2Model +from vllm.model_executor.models.interfaces import ( + HasInnerState, + IsHybrid, + SupportsMambaPrefixCaching, +) +from vllm.model_executor.models.nemotron_h import NemotronHForCausalLM, NemotronHModel from vllm.model_executor.models.utils import maybe_prefix from vllm.sequence import IntermediateTensors from vllm_omni.model_executor.models.output_templates import OmniOutput +from easymagpie_vllm_omni.backbone_patches import patch_silu_shared_experts from easymagpie_vllm_omni.config import EasyMagpieOmniArch from easymagpie_vllm_omni.local_transformer import EasyMagpieCodePredictor @@ -81,21 +93,18 @@ _DUMMY_TOKEN_ID = 0 -# ``dynamic_arg_dims`` is passed explicitly: this file uses -# ``from __future__ import annotations`` (PEP 563), so ``forward``'s annotations -# are strings and vLLM's annotation-based inference would fail with -# "No dynamic dimensions found...". These mirror vLLM's default inference -# (dim 0 for every tensor / IntermediateTensors argument). -@ignore_torch_compile -@support_torch_compile( - dynamic_arg_dims={ - "input_ids": 0, - "positions": 0, - "intermediate_tensors": 0, - "inputs_embeds": 0, - } -) -class EasyMagpieTTSForConditionalGeneration(nn.Module): +# NOTE: unlike the Qwen2 backbone variant, this class is *not* wrapped in +# ``@support_torch_compile``. The Nemotron-H backbone is a hybrid-Mamba model +# that manages its own ``torch.compile`` / CUDA-graph capture internally (as +# does :class:`EasyMagpieCodePredictor`), so the outer ``forward`` runs eagerly +# and dispatches into the two self-compiled subgraphs — matching the EasyMagpie +# vLLM sidecar (``EasyMagpieSmallMamba``). +class EasyMagpieTTSForConditionalGeneration( + nn.Module, + HasInnerState, + IsHybrid, + SupportsMambaPrefixCaching, +): """EasyMagpieTTS talker for vLLM-Omni. See the module docstring for the per-step flow and the per-request I/O @@ -104,6 +113,12 @@ class EasyMagpieTTSForConditionalGeneration(nn.Module): ``OmniGPUModelRunner``. """ + # Hybrid-Mamba bookkeeping (delegated to vLLM's NemotronH causal-LM, exactly + # like the EasyMagpie sidecar). vLLM expects these as class attributes. + get_mamba_state_dtype_from_config = NemotronHForCausalLM.get_mamba_state_dtype_from_config + get_mamba_state_shape_from_config = NemotronHForCausalLM.get_mamba_state_shape_from_config + get_mamba_state_copy_func = NemotronHForCausalLM.get_mamba_state_copy_func + # Omni runner hooks. has_preprocess: bool = True has_postprocess: bool = True @@ -129,11 +144,15 @@ def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: self.embedding_dim = arch.embedding_dim self.num_codebooks = arch.num_stacked_codebooks - # ── Backbone (reused vLLM text LM; fed via inputs_embeds) ─────── - self.backbone = Qwen2Model( + # ── Backbone (reused vLLM Nemotron-H LM; fed via inputs_embeds) ── + self.backbone = NemotronHModel( vllm_config=vllm_config, prefix=maybe_prefix(prefix, "backbone"), ) + # SmallMamba was trained with mlp_hidden_act=silu but vLLM's NemotronHMLP + # hard-codes ReLU² in shared_experts. Restore SiLU (no-op when the + # backbone has no MoE layers). + patch_silu_shared_experts(self.backbone) # ── Local transformer (its own compile group / CUDA graph) ────── with set_model_tag("local_transformer"): @@ -202,6 +221,43 @@ def _embed_phoneme(self, phoneme_tokens: torch.Tensor) -> torch.Tensor: # Decode-token dispatch (which positions need the local transformer) # ------------------------------------------------------------------ + @staticmethod + def _select_query_layout(attn_metadata): + """Return ``(max_query_len, query_start_loc)`` from heterogeneous metadata. + + The Nemotron-H backbone is hybrid, so ``attn_metadata`` is a per-layer + dict mixing two metadata types: + + * **attention** layers carry standard metadata that exposes the + batch-level ``max_query_len`` + ``query_start_loc`` (e.g. + ``TritonAttentionMetadata``); + * **Mamba2** layers carry ``Mamba2AttentionMetadata``, which has *no* + ``max_query_len`` and splits the query layout into ``query_start_loc_p`` + / ``query_start_loc_d`` instead. + + Both are built from the same batch query layout, so we prefer any + attention-layer metadata. As a fallback for a (hypothetical) attention-free + backbone, we infer a decode-only batch from the Mamba2 ``num_prefills`` + counter. Returns ``(None, None)`` when the layout can't be determined. + """ + metas = list(attn_metadata.values()) if isinstance(attn_metadata, dict) else [attn_metadata] + + # Preferred: an attention layer exposes the unified query layout. + for m in metas: + mql = getattr(m, "max_query_len", None) + qsl = getattr(m, "query_start_loc", None) + if mql is not None and qsl is not None: + return int(mql), qsl + + # Fallback: Mamba2-only backbone. We can at least detect a decode-only + # batch (every request contributes a single token) from the counters. + for m in metas: + if hasattr(m, "num_prefills") and hasattr(m, "num_decodes"): + if int(getattr(m, "num_prefills", 0)) == 0: + return 1, None # decode-only -> caller runs the LT everywhere + break + return None, None + def _get_decode_idxs(self): """Return ``(decode_token_indices, num_requests)`` for code-predictor dispatch. @@ -220,15 +276,12 @@ def _get_decode_idxs(self): if attn_metadata is None: return None, 0 - if isinstance(attn_metadata, dict): - any_layer_meta = next(iter(attn_metadata.values())) - else: - any_layer_meta = attn_metadata + max_query_len, start_loc = self._select_query_layout(attn_metadata) - if any_layer_meta.max_query_len == 1: + # Decode-only batch (or layout unavailable) -> run the LT on every token. + if max_query_len is None or max_query_len == 1 or start_loc is None: return None, 0 - start_loc = any_layer_meta.query_start_loc tokens_per_req = start_loc[1:] - start_loc[:-1] is_decode = tokens_per_req == 1 decode_token_indices = start_loc[:-1][is_decode] @@ -284,9 +337,9 @@ def forward( self._assemble_decode_embeddings(combined, valid) hidden_states = self.backbone( - input_ids, - positions, - intermediate_tensors, + input_ids=input_ids, + positions=positions, + intermediate_tensors=intermediate_tensors, inputs_embeds=combined, ) @@ -331,7 +384,10 @@ def _assemble_decode_embeddings(self, combined: torch.Tensor, idx) -> None: @torch.no_grad() def _predict_phonemes(self, hidden_states: torch.Tensor, idx) -> None: """Argmax the phoneme head and stash the prediction for the next step.""" - logits = self.phoneme_final_proj(hidden_states[idx].float()) + # Run in the model dtype (don't force fp32): ``phoneme_final_proj`` weights + # follow ``model_config.dtype`` (e.g. bf16), and argmax is dtype-insensitive, + # so an fp32 upcast here would mismatch the weight dtype in ``F.linear``. + logits = self.phoneme_final_proj(hidden_states[idx]) s = self.arch.phoneme_stacking_factor logits = logits.view(-1, s, self.arch.phoneme_vocab_size) self._dec_phoneme_tokens[idx] = logits.argmax(dim=-1).long() @@ -517,7 +573,8 @@ def postprocess(self, hidden_states: torch.Tensor, multimodal_outputs: Optional[ # Checkpoint prefixes (reference EasyMagpieTTS state dict) → in-model paths. # ``decoder.*`` is fed to the vLLM backbone loader separately (it understands - # HF Qwen2 naming + qkv packing). The TTS submodules are copied manually. + # HF Nemotron-H naming + Mamba/MoE packing). The TTS submodules are copied + # manually. _TTS_PREFIX_MAP = { "local_transformer.": "code_predictor.local_transformer.", "local_transformer_in_projection.": "code_predictor.local_transformer_in_projection.", @@ -538,14 +595,15 @@ def _remap_tts_key(self, name: str) -> Optional[str]: return None def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - """Load backbone (Qwen2) + TTS submodule weights from a converted checkpoint. + """Load backbone (Nemotron-H) + TTS submodule weights from a converted checkpoint. The converted checkpoint is expected to use the reference EasyMagpieTTS - key layout: the backbone under ``decoder.*`` (HF Qwen2 names) and the - TTS submodules at top level (``audio_embeddings.*``, ``local_transformer.*``, - ``phoneme_*``, ``text_embedding.*``, projection heads). Backbone weights - are routed to :meth:`Qwen2Model.load_weights` (which packs qkv / gate-up - and handles HF naming); TTS weights are copied directly by name. + key layout: the backbone under ``decoder.*`` (HF Nemotron-H names) and + the TTS submodules at top level (``audio_embeddings.*``, + ``local_transformer.*``, ``phoneme_*``, ``text_embedding.*``, projection + heads). Backbone weights are routed to :meth:`NemotronHModel.load_weights` + (which handles HF naming + Mamba/MoE packing); TTS weights are copied + directly by name. """ own_params = dict(self.named_parameters()) loaded: set[str] = set() @@ -578,9 +636,5 @@ def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: # Derived runtime state. self.code_predictor.init_forbidden_mask() - # The backbone's vestigial embed_tokens table is never consumed - # (everything goes through inputs_embeds); don't flag it as missing. - loaded.add("backbone.embed_tokens.weight") - logger.info("Loaded %d weights for EasyMagpieTTSForConditionalGeneration", len(loaded)) return loaded diff --git a/examples/tts/easymagpie_vllm_omni/pyproject.toml b/examples/tts/easymagpie_vllm_omni/pyproject.toml index 5cd41cead748..c6d4d8942c93 100644 --- a/examples/tts/easymagpie_vllm_omni/pyproject.toml +++ b/examples/tts/easymagpie_vllm_omni/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta" [project] name = "easymagpie-vllm-omni" version = "0.1.0" -description = "vLLM-Omni model definition for EasyMagpieTTS (Qwen2 backbone + AR local transformer)" +description = "vLLM-Omni model definition for EasyMagpieTTS (Nemotron-H hybrid-Mamba backbone + AR local transformer)" requires-python = ">=3.10" dependencies = [ # The heavy runtime deps (vllm, vllm-omni, torch) are provided by the From bb8b4276f347b0776ef1815d7d3bc4ef80d9c85f Mon Sep 17 00:00:00 2001 From: Viacheslav Klimkov Date: Tue, 2 Jun 2026 18:20:44 +0200 Subject: [PATCH 04/45] examples/tts/easymagpie_vllm_omni: make sure model runs with cuda graphs Signed-off-by: Viacheslav Klimkov --- .../easymagpie_vllm_omni/easymagpie_inference_demo.ipynb | 2 +- .../easymagpie_vllm_omni/local_transformer.py | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/examples/tts/easymagpie_vllm_omni/easymagpie_inference_demo.ipynb b/examples/tts/easymagpie_vllm_omni/easymagpie_inference_demo.ipynb index 88af6037e056..50e53a1c1e03 100644 --- a/examples/tts/easymagpie_vllm_omni/easymagpie_inference_demo.ipynb +++ b/examples/tts/easymagpie_vllm_omni/easymagpie_inference_demo.ipynb @@ -260,7 +260,7 @@ " \"model_arch\": \"EasyMagpieTTSForConditionalGeneration\",\n", " \"worker_type\": \"ar\",\n", " \"scheduler_cls\": \"vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler\",\n", - " \"enforce_eager\": True, # dummy run: skip CUDA-graph capture for a faster start\n", + " #\"enforce_eager\": True, # dummy run: skip CUDA-graph capture for a faster start\n", " \"trust_remote_code\": True,\n", " \"async_scheduling\": True,\n", " \"enable_prefix_caching\": False,\n", diff --git a/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/local_transformer.py b/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/local_transformer.py index a72ee6ecd52d..b48715604530 100644 --- a/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/local_transformer.py +++ b/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/local_transformer.py @@ -379,7 +379,11 @@ def generate_codes(self, dec_hidden: torch.Tensor) -> torch.Tensor: # Row 0: projected backbone hidden state (the AR "prompt"). buf[:, 0, :] = self.local_transformer_in_projection(dec_hidden) - forbidden = self.forbidden_mask if self.forbidden_mask.any() else None + # Always pass the mask unconditionally. An all-False mask makes + # ``masked_fill`` a no-op, so there's no need to guard with + # ``forbidden_mask.any()`` — and that guard is a data-dependent + # host sync that is illegal during CUDA-graph capture. + forbidden = self.forbidden_mask for k in range(self.num_codebooks): hidden = self(buf) # compiled transformer over the fixed buffer row = self.local_transformer_audio_out_projection(hidden[:, k, :]) From 999256918b1040e054014c5c591e8cfffab2182b Mon Sep 17 00:00:00 2001 From: Viacheslav Klimkov Date: Tue, 2 Jun 2026 18:20:44 +0200 Subject: [PATCH 05/45] examples/tts/easymagpie_vllm_omni: extend preprocess to take speaker embeddings and prepare prefill embeddings Signed-off-by: Viacheslav Klimkov --- .../easymagpie_inference_demo.ipynb | 106 +++++++--- .../easymagpie_vllm_omni/config.py | 7 + .../easymagpie_vllm_omni/easymagpie.py | 194 ++++++++++++++++-- 3 files changed, 261 insertions(+), 46 deletions(-) diff --git a/examples/tts/easymagpie_vllm_omni/easymagpie_inference_demo.ipynb b/examples/tts/easymagpie_vllm_omni/easymagpie_inference_demo.ipynb index 50e53a1c1e03..dd7322cef37c 100644 --- a/examples/tts/easymagpie_vllm_omni/easymagpie_inference_demo.ipynb +++ b/examples/tts/easymagpie_vllm_omni/easymagpie_inference_demo.ipynb @@ -16,10 +16,12 @@ "It follows the same `AsyncOmni` single-stage pattern as the reference\n", "`qwen3-tts` and `eartts` demos:\n", "\n", - "* **prefill** — the caller supplies a precomputed context embedding via\n", - " `additional_information.prompt_embeds` of shape `(T_ctx, embedding_dim)`, with\n", - " `prompt_token_ids = [0] * T_ctx` (exactly like qwen3-tts `talker_prompt_embeds`\n", - " / eartts `speaker_latent`).\n", + "* **prefill** — the caller supplies the speaker-encoded context-audio embedding\n", + " via `additional_information.speaker_embedding` `(T_audio, embedding_dim)` plus a\n", + " plain `context_text` string; the model assembles the full prefill context\n", + " (`[task_embedding? | speaker_embedding | context_text_embedded]`) and tokenizes\n", + " `context_text` itself. `prompt_token_ids = [0] * prompt_len`, sized with\n", + " `EasyMagpieTTSForConditionalGeneration.estimate_prompt_len(...)`.\n", "* **decode** — each step consumes one subword id from the streaming\n", " `additional_information.text_tokens` list; the local transformer samples all\n", " `C * S` stacked audio codebooks for the frame.\n", @@ -103,7 +105,11 @@ "supplied in the prompt.\n", "\n", "With `load_format=\\\"dummy\\\"` (set in the stage config) vLLM never reads weight\n", - "files, so a lone `config.json` is enough — no safetensors, no tokenizer." + "files, so no safetensors are needed. We do save the checkpoint's\n", + "text-conditioning tokenizer (`TEXT_TOKENIZER`, the Nemotron-H tokenizer that\n", + "matches `TEXT_VOCAB`) into the model dir, since the model tokenizes the\n", + "per-request `context_text` in-engine via\n", + "`AutoTokenizer.from_pretrained(model_path)`." ] }, { @@ -131,6 +137,10 @@ "PHONEME_STACKING = 1 # phoneme_stacking_factor\n", "PHONEME_VOCAB = 2051 # IPA-BPE 2048 tokenizer + 3 special tokens\n", "TEXT_VOCAB = 131072 # nemotron_h_config.vocab_size\n", + "# Text-conditioning tokenizer that matches the checkpoint (SmallMamba uses the\n", + "# Nemotron-H tokenizer, vocab 131072 == TEXT_VOCAB). Point this at the converted\n", + "# checkpoint dir / the checkpoint's tokenizer when running a real model.\n", + "TEXT_TOKENIZER = \"nvidia/Nemotron-H-8B-Base-8K\"\n", "\n", "config = {\n", " # Resolved through the `vllm.general_plugins` entry point registered by the\n", @@ -193,6 +203,14 @@ "\n", "MODEL_DIR = Path(tempfile.mkdtemp(prefix=\"easymagpie_dummy_\"))\n", "(MODEL_DIR / \"config.json\").write_text(json.dumps(config, indent=2))\n", + "\n", + "# The model tokenizes the per-request `context_text` string in-engine via\n", + "# `AutoTokenizer.from_pretrained(model_path)` (qwen3-tts style), so the model dir\n", + "# must ship the checkpoint's text-conditioning tokenizer. We save the matching\n", + "# Nemotron-H tokenizer (TEXT_TOKENIZER) into MODEL_DIR.\n", + "from transformers import AutoTokenizer\n", + "\n", + "AutoTokenizer.from_pretrained(TEXT_TOKENIZER, trust_remote_code=True).save_pretrained(MODEL_DIR)\n", "print(f\"Dummy model dir: {MODEL_DIR}\")\n", "\n", "# Sanity-check the arch the model will derive from this config.\n", @@ -235,8 +253,10 @@ "metadata": {}, "outputs": [], "source": [ - "T_CTX = 16 # prefill context-embedding length (prompt_token_ids = [0] * T_CTX)\n", "DECODE_STEPS = 32 # number of audio frames to decode\n", + "# Prefill length is derived at prompt-build time from the speaker embedding +\n", + "# tokenized context_text (see the prompt cell); these just need to be large\n", + "# enough to cover prefill + decode.\n", "MAX_MODEL_LEN = 512\n", "MAX_NUM_BATCHED_TOKENS = 512\n", "\n", @@ -311,18 +331,26 @@ "source": [ "## 3. Build the prompt\n", "\n", - "Two pieces of per-request input, passed through `additional_information`:\n", + "Per-request input, passed through `additional_information`:\n", "\n", - "* **`prompt_embeds`** `(T_ctx, embedding_dim)` — the precomputed context\n", - " embedding consumed during prefill. In a real run this is the speaker-encoded\n", - " context audio + context text produced by the caller; here we use random noise.\n", - " `prompt_token_ids = [0] * T_ctx` are placeholders (the model feeds the backbone\n", - " via `inputs_embeds`, never via these ids).\n", + "* **`speaker_embedding`** `(T_audio, embedding_dim)` — the speaker-encoded\n", + " context-audio embedding (the audio branch of `prepare_context_tensors`),\n", + " loaded here from `eng_speaker_emb.pt` (as written by\n", + " `easy_magpietts_extract_speaker_encoding.py`). The model assembles the full\n", + " prefill context itself as `[task_embedding? | speaker_embedding |\n", + " context_text_embedded]`.\n", + "* **`context_text`** — a plain conditioning string, here `\"[EN]\"`. The model\n", + " tokenizes it in-engine and embeds it through the baked `text_embedding` table.\n", "* **`text_tokens`** `list[int]` — the streaming subword stream; decode step `k`\n", " consumes `text_tokens[k]`. We provide one id per decode step.\n", "\n", - "(If the checkpoint had a phoneme branch you'd also stream `phoneme_tokens`; it's\n", - "disabled here via `phoneme_vocab_size = 0`.)" + "`prompt_token_ids = [0] * prompt_len` are placeholders (the model feeds the\n", + "backbone via `inputs_embeds`, never these ids). `prompt_len` must equal the\n", + "assembled context length, so we size it with the model's\n", + "`estimate_prompt_len(...)` — the length-only mirror of the in-engine prefill\n", + "assembly (à la qwen3-tts's `estimate_prompt_len_from_additional_information`).\n", + "\n", + "(If the checkpoint had a phoneme branch you'd also stream `phoneme_tokens`.)" ] }, { @@ -334,33 +362,61 @@ "source": [ "torch.manual_seed(0)\n", "\n", - "# Precomputed context embedding (random stand-in for the speaker/text encoder).\n", - "prompt_embeds = torch.randn(T_CTX, arch.embedding_dim, dtype=torch.float32)\n", + "from transformers import AutoTokenizer\n", + "\n", + "from easymagpie_vllm_omni.easymagpie import EasyMagpieTTSForConditionalGeneration\n", + "\n", + "# Speaker-encoded context audio (audio branch of prepare_context_tensors),\n", + "# produced by easy_magpietts_extract_speaker_encoding.py.\n", + "SPEAKER_EMB_FILE = \"eng_speaker_emb.pt\"\n", + "_loaded = torch.load(SPEAKER_EMB_FILE, map_location=\"cpu\")\n", + "speaker_embedding = _loaded[\"speaker_encoding\"] if isinstance(_loaded, dict) else _loaded\n", + "speaker_embedding = speaker_embedding.to(torch.float32)\n", + "\n", + "# Plain conditioning string; the model tokenizes + embeds it in-engine.\n", + "CONTEXT_TEXT = \"[EN]\"\n", + "\n", + "# Same tokenizer the engine loads from MODEL_DIR — used to size the prefill\n", + "# placeholders so prompt_token_ids length matches the assembled context.\n", + "tokenizer = AutoTokenizer.from_pretrained(MODEL_DIR, trust_remote_code=True)\n", + "prompt_len = EasyMagpieTTSForConditionalGeneration.estimate_prompt_len(\n", + " speaker_embedding,\n", + " tokenize=lambda t: tokenizer.encode(t),\n", + " context_text=CONTEXT_TEXT,\n", + " has_task_embedding=arch.num_task_embeddings > 0,\n", + ")\n", "\n", "# Streaming subword ids: one per decode step (step k consumes text_tokens[k]).\n", "text_tokens = torch.randint(0, TEXT_VOCAB, (DECODE_STEPS,)).tolist()\n", "\n", "additional_information = {\n", - " \"prompt_embeds\": prompt_embeds, # (T_ctx, embedding_dim) tensor\n", - " \"text_tokens\": text_tokens, # list[int], grows by one per step\n", + " \"speaker_embedding\": speaker_embedding, # (T_audio, embedding_dim) tensor\n", + " \"context_text\": CONTEXT_TEXT, # plain string, tokenized in-model\n", + " \"text_tokens\": text_tokens, # list[int], grows by one per step\n", "}\n", "\n", "prompt = {\n", - " \"prompt_token_ids\": [0] * T_CTX, # prefill placeholders\n", + " \"prompt_token_ids\": [0] * prompt_len, # prefill placeholders\n", " \"additional_information\": additional_information,\n", "}\n", "\n", + "assert prompt_len + DECODE_STEPS <= MAX_MODEL_LEN, (\n", + " f\"prompt_len ({prompt_len}) + decode steps ({DECODE_STEPS}) exceeds \"\n", + " f\"MAX_MODEL_LEN ({MAX_MODEL_LEN}); raise MAX_MODEL_LEN / MAX_NUM_BATCHED_TOKENS.\"\n", + ")\n", + "\n", + "print(f\"speaker_embedding : {tuple(speaker_embedding.shape)}\")\n", + "print(f\"context_text : {CONTEXT_TEXT!r} -> {tokenizer.encode(CONTEXT_TEXT)}\")\n", + "print(f\"prompt_len (placeholders) : {prompt_len}\")\n", + "print(f\"decode steps (max_tokens) : {DECODE_STEPS}\")\n", + "print(f\"text_tokens[:8] : {text_tokens[:8]}\")\n", + "\n", "sampling_params = SamplingParams(\n", " temperature=0.0,\n", " max_tokens=DECODE_STEPS,\n", " detokenize=False,\n", " ignore_eos=True, # dummy logits never emit a meaningful EOS -> run the full budget\n", - ")\n", - "\n", - "print(f\"T_ctx (prefill placeholders) : {T_CTX}\")\n", - "print(f\"prompt_embeds : {tuple(prompt_embeds.shape)}\")\n", - "print(f\"decode steps (max_tokens) : {DECODE_STEPS}\")\n", - "print(f\"text_tokens[:8] : {text_tokens[:8]}\")" + ")" ] }, { diff --git a/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/config.py b/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/config.py index cdba51613a8c..1b086ec3e562 100644 --- a/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/config.py +++ b/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/config.py @@ -72,6 +72,12 @@ class EasyMagpieOmniArch: phoneme_stacking_factor: int = 1 phoneme_vocab_size: int = 2051 + # Number of multi-mode task ("service token") embeddings. The reference model + # prepends a single learned per-mode embedding to the prefill context when + # trained with >1 mode (``cfg.training_modes``); 0 disables it (single-mode + # checkpoints have no ``task_embedding`` table). + num_task_embeddings: int = 0 + local_transformer_n_layers: int = 3 local_transformer_n_heads: int = 12 local_transformer_hidden_dim: int = 1536 @@ -136,6 +142,7 @@ def from_hf_config(cls, hf_config: Any) -> "EasyMagpieOmniArch": "frame_stacking_factor", "phoneme_stacking_factor", "phoneme_vocab_size", + "num_task_embeddings", "local_transformer_n_layers", "local_transformer_n_heads", "local_transformer_hidden_dim", diff --git a/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/easymagpie.py b/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/easymagpie.py index e188b9387a7b..cb5e8bd346f4 100644 --- a/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/easymagpie.py +++ b/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/easymagpie.py @@ -47,10 +47,25 @@ Per-request I/O (via ``additional_information``): -* ``prompt_embeds`` (prefill only) — ``(T_ctx, embedding_dim)`` precomputed - context/prompt embedding (speaker-encoded context audio + context text) - produced by the caller, exactly like qwen3-tts ``talker_prompt_embeds`` / - eartts ``speaker_latent``. The user passes ``prompt_token_ids = [0] * T_ctx``. +* ``speaker_embedding`` (prefill only) — ``(T_audio, embedding_dim)`` speaker- + encoded context-audio embedding (the audio branch of the reference + ``prepare_context_tensors``), e.g. the tensor saved by + ``easy_magpietts_extract_speaker_encoding.py``. ``preprocess`` assembles the + full prefill context embedding itself as + ``[task_embedding | speaker_embedding | context_text_embedded]`` — the same + layout the reference model builds — so the caller only does the speaker-encoder + math and passes plain context text (the model tokenizes + embeds it and + prepends the per-mode service token). +* ``context_text`` (prefill only, optional) — plain conditioning string (e.g. + ``"[EN]"``); tokenized in-model with the checkpoint's text tokenizer and + embedded through the baked per-subword ``text_embedding`` table. Defaults to + ``"[NO TEXT CONTEXT]"`` when omitted. +* ``task_mode_id`` (prefill only, optional) — int selecting the per-mode task + ("service token") embedding row; defaults to ``0``. Ignored for single-mode + checkpoints (no ``task_embedding`` table). + + The caller passes ``prompt_token_ids = [0] * T_ctx``, where ``T_ctx`` is the + assembled context length (``[task?] + T_audio + len(tokenize(context_text))``). * ``text_tokens`` — Python ``list[int]`` of subword ids that grows by one per decode step; step ``k`` consumes ``text_tokens[k]`` (embedded through the precomputed per-subword table). @@ -60,7 +75,7 @@ from __future__ import annotations import bisect -from collections.abc import Iterable +from collections.abc import Callable, Iterable from typing import Any, Optional import torch @@ -92,6 +107,9 @@ # argmax-at-0 dummy logits, so this only needs to be a valid id. _DUMMY_TOKEN_ID = 0 +# Context text used when the request omits ``context_text`` +_DEFAULT_CONTEXT_TEXT = "[EN]" + # NOTE: unlike the Qwen2 backbone variant, this class is *not* wrapped in # ``@support_torch_compile``. The Nemotron-H backbone is a hybrid-Mamba model @@ -171,6 +189,22 @@ def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: text_vocab_size = int(getattr(hf_config, "text_vocab_size", getattr(hf_config, "vocab_size", 0))) self.text_embedding = nn.Embedding(text_vocab_size, self.embedding_dim) + # Task ("service token") embedding — a single learned per-mode row the + # reference model prepends to the prefill context when trained with >1 + # mode. Built only when the checkpoint carries one; otherwise ``None``. + self.num_task_embeddings = int(arch.num_task_embeddings) + if self.num_task_embeddings > 0: + self.task_embedding = nn.Embedding(self.num_task_embeddings, self.embedding_dim) + else: + self.task_embedding = None + + # Context-text tokenizer, loaded lazily from the model directory (same + # ``AutoTokenizer.from_pretrained(model_path)`` pattern as qwen3-tts). It + # turns the per-request ``context_text`` string (e.g. ``"[EN]"``) into the + # subword ids that the baked ``text_embedding`` table consumes — so the + # caller passes plain text, never pre-tokenized ids. + self._text_tokenizer: Any = None + # Phoneme channel (optional — only built when the checkpoint has one). self.has_phoneme = arch.phoneme_vocab_size > 0 and arch.phoneme_stacking_factor > 0 if self.has_phoneme: @@ -432,10 +466,13 @@ def make_omni_output(self, model_outputs, **_: Any) -> OmniOutput: # ------------------------------------------------------------------ @staticmethod - def _unwrap(value: Any) -> Any: + def _first_str(value: Any) -> str: + """Return the first element of a list-wrapped scalar, or the scalar itself, as a string.""" if isinstance(value, list): - return value[0] if value else None - return value + return str(value[0]) if value else "" + if value is None: + return "" + return str(value) def preprocess( self, @@ -448,9 +485,11 @@ def preprocess( ) -> tuple[torch.Tensor, torch.Tensor, dict[str, Any]]: """Build per-request ``(input_ids, inputs_embeds)`` for this step. - Prefill (``span_len > 1``): slice the precomputed ``prompt_embeds`` - context embedding into this chunk and return it; ``input_ids`` are - placeholders. Decode (``span_len == 1``): write the per-token decode + Prefill (``span_len > 1``): assemble the full context embedding + (``[task_embedding | speaker_embedding | context_text_embedded]`` from + the per-request inputs; see :meth:`_build_prefill_embeds`), slice this + chunk out of it, and return it; + ``input_ids`` are placeholders. Decode (``span_len == 1``): write the per-token decode inputs (previous codes, current text token, previous phoneme) into the model buffers at ``start`` and return a zero embedding that :meth:`forward` accumulates into. @@ -479,25 +518,19 @@ def _preprocess_prefill( device: torch.device, info_dict: dict[str, Any], ) -> tuple[torch.Tensor, torch.Tensor, dict[str, Any]]: - prompt_embeds = self._unwrap(info_dict.get("prompt_embeds")) - if not isinstance(prompt_embeds, torch.Tensor) or prompt_embeds.ndim != 2: - raise ValueError( - "EasyMagpieTTS preprocess requires additional_information.prompt_embeds " - "of shape (T_ctx, embedding_dim) for prefill." - ) - prompt_embeds = prompt_embeds.to(device=device, dtype=self._combined_embeddings.dtype) + prefill_embeds = self._build_prefill_embeds(device, info_dict) offset = int(info_dict.get("ear_prefill_offset", 0) or 0) - total = int(prompt_embeds.shape[0]) + total = int(prefill_embeds.shape[0]) s = max(0, min(offset, total)) e = max(0, min(offset + span_len, total)) - take = prompt_embeds[s:e] + take = prefill_embeds[s:e] if int(take.shape[0]) < span_len: pad_n = span_len - int(take.shape[0]) pad_rows = ( take[-1:].expand(pad_n, -1) if take.shape[0] > 0 - else prompt_embeds.new_zeros(pad_n, prompt_embeds.shape[-1]) + else prefill_embeds.new_zeros(pad_n, prefill_embeds.shape[-1]) ) take = torch.cat([take, pad_rows], dim=0) @@ -508,6 +541,121 @@ def _preprocess_prefill( input_ids_out = torch.full_like(input_ids, _DUMMY_TOKEN_ID) return input_ids_out, take, info_update + def _build_prefill_embeds( + self, + device: torch.device, + info_dict: dict[str, Any], + ) -> torch.Tensor: + """Assemble the full ``(T_ctx, embedding_dim)`` prefill context embedding. + + Reproduces the prefill assembly from the reference + ``prepare_context_tensors``:: + + [task_embedding | speaker_embedding | context_text_embedded] + + from the per-request inputs: + + * ``speaker_embedding`` — the speaker-encoded context-audio embedding + (e.g. produced by ``easy_magpietts_extract_speaker_encoding.py``), + required as a 2-D ``(T_audio, embedding_dim)`` tensor. + * ``context_text`` — a plain string (e.g. ``"[EN]"``); tokenized in-model + (see :meth:`_encode_context_text`) and embedded through the baked + per-subword ``text_embedding`` table (which already folds in the CAS + encoder, matching the default ``disable_cas_for_context_text=False`` + training). Defaults to ``"[NO TEXT CONTEXT]"`` when omitted. + * ``task_mode_id`` — selects the per-mode task ("service token") + embedding row; prepended only when the checkpoint has a task table. + + Returns the full context embedding; the per-chunk slicing/padding is done + by :meth:`_preprocess_prefill`. + """ + dtype = self._combined_embeddings.dtype + + speaker_embedding = info_dict.get("speaker_embedding") + assert isinstance(speaker_embedding, torch.Tensor) and speaker_embedding.ndim == 2, ( + "EasyMagpieTTS preprocess expects additional_information.speaker_embedding to be a 2-D " + "(T_audio, embedding_dim) tensor (the speaker-encoded context audio); " + f"got {type(speaker_embedding).__name__}" + + (f" with ndim={speaker_embedding.ndim}" if isinstance(speaker_embedding, torch.Tensor) else "") + ) + + parts: list[torch.Tensor] = [] + + # Task / "service token" embedding (prepended), when present. + if self.task_embedding is not None: + task_mode_id = int(info_dict.get("task_mode_id", 0) or 0) + task_mode_id = max(0, min(task_mode_id, self.num_task_embeddings - 1)) + task_row = self.task_embedding(torch.tensor([task_mode_id], device=device, dtype=torch.long)) + parts.append(task_row.to(dtype)) + + # Speaker-encoded context audio. + parts.append(speaker_embedding.to(device=device, dtype=dtype)) + + # Context text: tokenized in-model and embedded through the baked table. + context_text = self._first_str(info_dict.get("context_text")) or _DEFAULT_CONTEXT_TEXT + ctx_ids = self._encode_context_text(context_text, device) + if ctx_ids.numel() > 0: + parts.append(self.text_embedding(ctx_ids).to(dtype)) + + return torch.cat(parts, dim=0) + + def _get_text_tokenizer(self): + """Lazily load the context-text tokenizer from the model directory. + + Mirrors qwen3-tts: the converted checkpoint ships a HuggingFace + ``AutoTokenizer`` (the model's text-conditioning tokenizer) alongside its + weights, so we load it on first use from ``model_path``. + """ + if self._text_tokenizer is None: + from transformers import AutoTokenizer + + self._text_tokenizer = AutoTokenizer.from_pretrained(self.model_path, trust_remote_code=True) + return self._text_tokenizer + + def _encode_context_text(self, context_text: str, device: torch.device) -> torch.Tensor: + """Tokenize ``context_text`` to subword ids (matching the reference encode path). + + The reference ``AggregatedTTSTokenizer.encode`` calls the underlying + HF tokenizer's ``encode`` (default ``add_special_tokens``) for the + text-conditioning tokenizer, which sits at offset 0 in the aggregate, so + its raw ids index the baked ``text_embedding`` table directly. + """ + tok = self._get_text_tokenizer() + ids = tok.encode(context_text) + return torch.tensor(ids, device=device, dtype=torch.long) + + @staticmethod + def estimate_prompt_len( + speaker_embedding: torch.Tensor, + *, + tokenize: Callable[[str], Iterable[int]], + context_text: str = _DEFAULT_CONTEXT_TEXT, + has_task_embedding: bool = False, + ) -> int: + """Length-only mirror of :meth:`_build_prefill_embeds` (à la qwen3-tts's + ``estimate_prompt_len_from_additional_information``). + + The engine assembles the prefill context as + ``[task_embedding? | speaker_embedding | context_text_embedded]``, so the + caller must pass ``prompt_token_ids = [0] * estimate_prompt_len(...)`` for + the placeholder length to match the assembled embedding length (otherwise + vLLM pads / truncates and quality drops). + + Args: + speaker_embedding: ``(T_audio, embedding_dim)`` speaker-encoded + context-audio embedding (only its length is used). + tokenize: callable turning ``context_text`` into its subword ids + (e.g. ``lambda t: tokenizer.encode(t)``) — must match the + tokenizer the engine loads from ``model_path``. + context_text: conditioning string (default ``"[NO TEXT CONTEXT]"``). + has_task_embedding: whether the checkpoint prepends a task / + "service token" embedding (``num_task_embeddings > 0``). + """ + t_audio = int(speaker_embedding.shape[0]) + ctx_len = len(list(tokenize(context_text or _DEFAULT_CONTEXT_TEXT))) + task_len = 1 if has_task_embedding else 0 + return task_len + t_audio + ctx_len + def _preprocess_decode( self, input_ids: torch.Tensor, @@ -585,6 +733,7 @@ def postprocess(self, hidden_states: torch.Tensor, multimodal_outputs: Optional[ "phoneme_embeddings.": "phoneme_embeddings.", "phoneme_final_proj.": "phoneme_final_proj.", "text_embedding.": "text_embedding.", + "task_embedding.": "task_embedding.", } def _remap_tts_key(self, name: str) -> Optional[str]: @@ -617,6 +766,9 @@ def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: if mapped is None: # Unrelated checkpoint section (codec, speaker encoder, CAS, etc.). continue + if mapped.startswith("task_embedding.") and self.task_embedding is None: + # Single-mode model: checkpoint may still ship an (unused) table. + continue target = own_params.get(mapped) if target is None: logger.warning("EasyMagpieTTS: no parameter for checkpoint key %s -> %s", name, mapped) From 3a8d50b36398cf7d96dcce95f81d9772edd44f95 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimkov Date: Tue, 2 Jun 2026 18:20:44 +0200 Subject: [PATCH 06/45] examples/tts/easymagpie_vllm_omni: introduce script to convert the checkpoint to vllm omni one Signed-off-by: Viacheslav Klimkov --- .../easy_magpietts_convert_to_vllm.py | 438 ++++++++++++++++++ .../easymagpie_inference_demo.ipynb | 367 ++++++++------- .../easymagpie_vllm_omni/easymagpie.py | 19 +- 3 files changed, 658 insertions(+), 166 deletions(-) create mode 100644 examples/tts/easymagpie_vllm_omni/easy_magpietts_convert_to_vllm.py diff --git a/examples/tts/easymagpie_vllm_omni/easy_magpietts_convert_to_vllm.py b/examples/tts/easymagpie_vllm_omni/easy_magpietts_convert_to_vllm.py new file mode 100644 index 000000000000..664a243d7415 --- /dev/null +++ b/examples/tts/easymagpie_vllm_omni/easy_magpietts_convert_to_vllm.py @@ -0,0 +1,438 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Convert an EasyMagpieTTS ``.nemo`` checkpoint to a vLLM-Omni model directory. + +The output directory is self-contained and ready to be passed as ``model=`` +to the ``easymagpie_vllm_omni`` vLLM-Omni model +(:class:`easymagpie_vllm_omni.easymagpie.EasyMagpieTTSForConditionalGeneration`). +It contains: + +* ``config.json`` — the flat HF-style config the vLLM model reads at + construction (the Nemotron-H backbone fields + the EasyMagpie scalars consumed + by :class:`easymagpie_vllm_omni.config.EasyMagpieOmniArch`). +* ``model.safetensors`` (+ ``model.safetensors.index.json``) — the converted + weights using the reference EasyMagpieTTS key layout expected by the vLLM + model's ``load_weights`` (``decoder.*`` backbone + top-level TTS submodules). +* the checkpoint's **text-conditioning tokenizer** saved via + ``AutoTokenizer.save_pretrained`` so the model can tokenize per-request + ``context_text`` in-engine. +* ``speaker_embeddings/.pt`` (optional) — pre-computed speaker-encoder + outputs for one or more reference audio files, used as the ``speaker_embedding`` + input at inference time. + +Compared to running the reference model, the character-aware subword (CAS) +encoder is collapsed into a single pre-computed lookup table mapping +``subword_id -> embedding`` (the CAS encoder is fully deterministic per subword +id, so it is baked once at conversion time and never run inside the engine). The +``decoder``'s unused token-embedding table is replaced by a tiny dummy (the +backbone is always fed via ``inputs_embeds``). + +Example:: + + python examples/tts/easymagpie_vllm_omni/easy_magpietts_convert_to_vllm.py \\ + --nemo_file /path/to/EMTTS_SmallMamba.nemo \\ + --codec_model_path /path/to/25fps_spectral_codec.nemo \\ + --outdir ./easymagpie_vllm_model \\ + --context_audio /path/to/reference_voice.wav --speaker_name eng +""" +from __future__ import annotations + +import argparse +import json +import os + +import torch +import tqdm +from omegaconf import OmegaConf +from safetensors.torch import save_file + +from nemo.collections.tts.modules.magpietts_inference.utils import ModelLoadConfig, load_easy_magpie_model +from nemo.collections.tts.modules.magpietts_modules import add_special_tokens +from nemo.utils import logging + +# Top-level checkpoint key prefixes the vLLM model's ``load_weights`` consumes +# for the TTS submodules (everything else under these names maps 1:1 into the +# vLLM model). ``text_embedding.*`` is intentionally excluded here: it is +# replaced by the pre-computed per-subword lookup table. +_TTS_PREFIXES = ( + "audio_embeddings.", + "audio_in_projection.", + "local_transformer.", + "local_transformer_in_projection.", + "local_transformer_audio_out_projection.", + "local_transformer_out_projections.", + "phoneme_embeddings.", + "phoneme_final_proj.", + "task_embedding.", +) + +# The backbone token-embedding table is never consumed at runtime (the model +# runs off ``inputs_embeds``), so we ship a dummy table. It must still be >= 2: +# vLLM's profiling ``_dummy_sampler_run`` sets ``top_k = vocab_size - 1`` and then +# gathers at index ``vocab_size - top_k``, which is out of bounds for a width-1 +# logits tensor (device-side "scatter gather index out of bounds" assert). +_BACKBONE_VOCAB_SIZE = 2 + +# Nemotron-H backbone config fields forwarded into the flat vLLM ``config.json``. +# Names match the HF/vLLM Nemotron-H config (and the NeMo ``NemotronHConfig``). +_NEMOTRON_CONFIG_FIELDS = ( + "hidden_size", + "num_hidden_layers", + "num_attention_heads", + "num_key_value_heads", + "head_dim", + "attention_dropout", + "attention_bias", + "max_position_embeddings", + "mamba_num_heads", + "mamba_head_dim", + "ssm_state_size", + "conv_kernel", + "n_groups", + "chunk_size", + "mamba_hidden_act", + "use_conv_bias", + "use_bias", + "intermediate_size", + "mlp_hidden_act", + "mlp_bias", + "n_routed_experts", + "num_experts_per_tok", + "moe_intermediate_size", + "moe_shared_expert_intermediate_size", + "n_group", + "topk_group", + "routed_scaling_factor", + "norm_topk_prob", + "hybrid_override_pattern", + "layer_norm_epsilon", + "residual_in_fp32", +) + + +def parse_args(): + parser = argparse.ArgumentParser( + description="Convert an EasyMagpieTTS .nemo checkpoint to a vLLM-Omni model directory." + ) + parser.add_argument("--nemo_file", required=True, help="Path to the EasyMagpieTTS .nemo checkpoint.") + parser.add_argument("--codec_model_path", required=True, help="Path to the audio codec .nemo checkpoint.") + parser.add_argument("--outdir", required=True, help="Output directory for the vLLM model.") + parser.add_argument( + "--phoneme_tokenizer_path", + default=None, + help="Override the phoneme (IPA BPE) tokenizer path baked into the checkpoint.", + ) + parser.add_argument( + "--disable_cas_for_context_text", + action="store_true", + help="Set for legacy checkpoints trained without CAS embeddings on context text.", + ) + parser.add_argument( + "--text_tokenizer", + default=None, + help="HuggingFace tokenizer name/path to export. Defaults to the checkpoint's " + "text-conditioning AutoTokenizer (`pretrained_model`).", + ) + parser.add_argument( + "--context_audio", + default=None, + help="Optional reference wav for which to pre-compute a speaker embedding.", + ) + parser.add_argument( + "--speaker_name", + default="default", + help="Name for the saved speaker embedding (speaker_embeddings/.pt).", + ) + parser.add_argument("--context_audio_duration", type=float, default=5.0) + parser.add_argument( + "--dtype", + default="bfloat16", + choices=["bfloat16", "float16", "float32"], + help="Saved weight dtype / config torch_dtype. bf16 matches the reference inference setup.", + ) + parser.add_argument( + "--precompute_batch_size", + type=int, + default=1024, + help="Batch size for pre-computing per-subword text embeddings.", + ) + parser.add_argument("--device", default="cuda") + return parser.parse_args() + + +@torch.no_grad() +def precompute_text_embeddings(model, batch_size: int) -> torch.Tensor: + """Bake the per-subword text embedding into a single lookup table. + + Runs ``embed_text_tokens`` (decoder subword embedding + the deterministic + char-aware subword encoder) once per subword id so the vLLM model can replace + the whole text-embedding path with a single ``nn.Embedding`` lookup. + + Returns: + Tensor of shape ``[vocab_size, embedding_dim]`` (float32). + """ + device = next(model.parameters()).device + + # Vocabulary size of the subword id space (decoder text-embedding table when + # present; otherwise the CAS-only id range, which ends at cfg_unk_token_id). + if getattr(model, "text_embedding", None) is not None: + vocab_size = model.text_embedding.num_embeddings + else: + vocab_size = int(model.cfg_unk_token_id) + 1 + embedding_dim = int(model.cfg.embedding_dim) + + table = torch.zeros((vocab_size, embedding_dim), dtype=torch.float32, device=device) + logging.info(f"Pre-computing text embeddings for {vocab_size} subword ids on {device}") + for start in tqdm.tqdm(range(0, vocab_size, batch_size), desc="Pre-computing text embeddings"): + end = min(start + batch_size, vocab_size) + ids = torch.arange(start, end, dtype=torch.long, device=device).unsqueeze(0) # (1, n) + lens = torch.tensor([end - start], dtype=torch.long, device=device) + embeds = model.embed_text_tokens(ids, text_lens=lens, disable_cas_embedding=False) # (1, n, E) + table[start:end] = embeds.squeeze(0).to(torch.float32) + return table.cpu() + + +@torch.no_grad() +def extract_speaker_embedding(model, context_audio_path: str, context_audio_duration: float) -> torch.Tensor: + """Reproduce the audio branch of ``prepare_context_tensors`` for one wav. + + Mirrors ``easy_magpietts_extract_speaker_encoding.py``: encode the (trimmed) + reference audio to codec codes, add special tokens, frame-stack, embed the + per-codebook tokens, and (when enabled) run the speaker encoder. Returns the + ``(T_audio, embedding_dim)`` tensor consumed as the model's ``speaker_embedding``. + """ + device = next(model.parameters()).device + + context_audio = model._load_audio_for_inference(context_audio_path, model.sample_rate) + context_audio = model._adjust_audio_to_duration_for_inference( + context_audio, + model.sample_rate, + context_audio_duration, + model.codec_model_samples_per_frame, + ) + context_audio = context_audio.to(device) + context_audio_lens = torch.tensor([context_audio.size(1)], dtype=torch.long, device=device) + context_audio_codes, context_audio_codes_lens = model._codec_helper.audio_to_codes( + context_audio, context_audio_lens + ) + + if model._codec_converter is not None: + context_audio_codes = model._codec_converter.convert_original_to_new( + audio_tokens=context_audio_codes, audio_lens=context_audio_codes_lens + ).long() + + context_audio_codes, context_audio_codes_lens = add_special_tokens( + codes=context_audio_codes, + codes_len=context_audio_codes_lens, + bos_id=model.context_audio_bos_id, + eos_id=model.context_audio_eos_id, + ) + context_audio_codes, context_audio_codes_lens = model.stack_codes( + context_audio_codes, + context_audio_codes_lens, + model.context_audio_bos_id, + model.context_audio_eos_id, + model.frame_stacking_factor, + model.num_audio_codebooks, + ) + + context_audio_embedded = model.embed_audio_tokens(context_audio_codes) # (B, T_audio, E) + if getattr(model, "use_speaker_encoder", False): + context_audio_embedded = model.encode_context_audio_embeddings( + context_audio_embedded=context_audio_embedded, + context_audio_lens=context_audio_codes_lens, + ) + else: + logging.warning( + "Checkpoint has use_speaker_encoder=False; saving raw per-codebook audio embeddings " + "(no speaker encoder applied)." + ) + + audio_len = int(context_audio_codes_lens[0].item()) + return context_audio_embedded[0, :audio_len].contiguous().float().detach().cpu() + + +def build_config(model, vocab_size: int, torch_dtype: str) -> dict: + """Build the flat vLLM ``config.json`` dict from the loaded NeMo model.""" + from nemo.collections.tts.modules.nemotron_h_decoder import NemotronHConfig + + cfg = model.cfg + if cfg.get("decoder_type", "huggingface") != "nemotron_h": + raise ValueError( + "The easymagpie_vllm_omni model only supports a Nemotron-H backbone " + f"(decoder_type='nemotron_h'); got '{cfg.get('decoder_type')}'." + ) + + hidden_dim = int(cfg.hidden_dim) + embedding_dim = int(cfg.embedding_dim) + + # Resolve the backbone config exactly as NeMo does (fills head_dim, expands + # the hybrid pattern to num_hidden_layers, etc.). + nemotron_dict = dict(OmegaConf.to_container(cfg.nemotron_h_config, resolve=True)) + nemotron_dict.setdefault("hidden_size", embedding_dim) + nemotron_cfg = NemotronHConfig(**nemotron_dict) + + config: dict = {"architectures": ["EasyMagpieTTSForConditionalGeneration"], "model_type": "nemotron_h"} + for field in _NEMOTRON_CONFIG_FIELDS: + if hasattr(nemotron_cfg, field): + config[field] = getattr(nemotron_cfg, field) + config["tie_word_embeddings"] = False + config["torch_dtype"] = torch_dtype + # The backbone token-embedding table is never consumed (inputs_embeds path); + # the dummy logits width follows it. Must be >= 2 (see ``_BACKBONE_VOCAB_SIZE``). + # The text path is driven by ``text_vocab_size`` / the baked ``text_embedding`` + # table instead. + config["vocab_size"] = _BACKBONE_VOCAB_SIZE + + # ── EasyMagpie scalars (read by EasyMagpieOmniArch.from_hf_config) ── + config["text_vocab_size"] = vocab_size + config["embedding_dim"] = embedding_dim + config["audio_embedding_dim"] = int(cfg.get("audio_embedding_dim", hidden_dim)) + config["num_audio_codebooks"] = int(model.num_audio_codebooks) + config["codebook_size"] = int(model.codebook_size) + config["frame_stacking_factor"] = int(model.frame_stacking_factor) + + has_phoneme = getattr(model, "phoneme_tokenizer", None) is not None + config["phoneme_stacking_factor"] = int(getattr(model, "phoneme_stacking_factor", 0)) if has_phoneme else 0 + config["phoneme_vocab_size"] = int(getattr(model, "phoneme_vocab_size", 0)) if has_phoneme else 0 + + config["num_task_embeddings"] = len(model.training_modes) if model.task_embedding is not None else 0 + + config["local_transformer_n_layers"] = int(cfg.get("local_transformer_n_layers", 2)) + config["local_transformer_n_heads"] = int(cfg.get("local_transformer_n_heads", 1)) + config["local_transformer_hidden_dim"] = int(cfg.get("local_transformer_hidden_dim", hidden_dim)) + + # Pin the exact special-token ids (covers legacy ``forced_*`` checkpoints). + config["forced_audio_bos_id"] = int(model.audio_bos_id) + config["forced_audio_eos_id"] = int(model.audio_eos_id) + config["forced_mask_token_id"] = int(model.mask_token_id) + + return config + + +def select_weights(state_dict: dict, hidden_dim: int, dtype: torch.dtype) -> dict: + """Select + rename checkpoint weights into the vLLM ``load_weights`` layout.""" + weights: dict = {} + + # Backbone: keep all ``decoder.*`` except the unused token-embedding table. + for key, value in state_dict.items(): + if not key.startswith("decoder."): + continue + if key == "decoder.embeddings.weight": + continue + if key.endswith(".causal_mask"): + continue + weights[key] = value.to(dtype) if value.is_floating_point() else value + + # Dummy backbone embeddings (size ``_BACKBONE_VOCAB_SIZE``) — never consumed + # at runtime; sized to match ``config.vocab_size``. + weights["decoder.embeddings.weight"] = torch.zeros(_BACKBONE_VOCAB_SIZE, hidden_dim, dtype=dtype) + + # TTS submodules copied 1:1. + for key, value in state_dict.items(): + if key.endswith(".causal_mask"): + continue + if any(key.startswith(prefix) for prefix in _TTS_PREFIXES): + weights[key] = value.to(dtype) if value.is_floating_point() else value + + return weights + + +def save_text_tokenizer(model, outdir: str, override: str | None) -> None: + """Export the checkpoint's text-conditioning tokenizer into ``outdir``.""" + from transformers import AutoTokenizer + + pretrained = override + if pretrained is None: + tok_name = model.text_conditioning_tokenizer_name + tok_cfg = model.cfg.text_tokenizers[tok_name] + if tok_cfg.get("_target_", None) != "AutoTokenizer" or tok_cfg.get("pretrained_model", None) is None: + raise ValueError( + "Could not infer the text-conditioning AutoTokenizer from the checkpoint config. " + "Pass --text_tokenizer explicitly." + ) + pretrained = tok_cfg.pretrained_model + + logging.info(f"Saving text tokenizer '{pretrained}' to {outdir}") + AutoTokenizer.from_pretrained(pretrained, trust_remote_code=True).save_pretrained(outdir) + + +def convert(args) -> None: + os.makedirs(args.outdir, exist_ok=True) + dtype = {"bfloat16": torch.bfloat16, "float16": torch.float16, "float32": torch.float32}[args.dtype] + + model, ckpt_name = load_easy_magpie_model( + ModelLoadConfig( + nemo_file=args.nemo_file, + codecmodel_path=args.codec_model_path, + phoneme_tokenizer_path=args.phoneme_tokenizer_path, + disable_cas_for_context_text=args.disable_cas_for_context_text, + ), + device=args.device, + ) + logging.info(f"Loaded EasyMagpieTTS checkpoint: {ckpt_name}") + + hidden_dim = int(model.cfg.hidden_dim) + + # ── 1. Pre-compute the per-subword text embedding table ────────────── + text_table = precompute_text_embeddings(model, args.precompute_batch_size) + vocab_size = int(text_table.shape[0]) + + # ── 2. config.json ─────────────────────────────────────────────────── + config = build_config(model, vocab_size, args.dtype) + with open(os.path.join(args.outdir, "config.json"), "w") as f: + json.dump(config, f, indent=2) + logging.info("Saved config.json") + + # ── 3. weights ─────────────────────────────────────────────────────── + state_dict = model.state_dict() + weights = select_weights(state_dict, hidden_dim, dtype) + weights["text_embedding.weight"] = text_table.to(dtype) + + safetensors_path = os.path.join(args.outdir, "model.safetensors") + save_file(weights, safetensors_path, metadata={"format": "pt"}) + index = { + "metadata": {"total_size": sum(w.numel() * w.element_size() for w in weights.values())}, + "weight_map": {name: "model.safetensors" for name in weights}, + } + with open(os.path.join(args.outdir, "model.safetensors.index.json"), "w") as f: + json.dump(index, f, indent=2) + logging.info(f"Saved {len(weights)} weights to {safetensors_path}") + + # ── 4. text tokenizer ──────────────────────────────────────────────── + save_text_tokenizer(model, args.outdir, args.text_tokenizer) + + # ── 5. optional speaker embedding ──────────────────────────────────── + if args.context_audio is not None: + speaker_dir = os.path.join(args.outdir, "speaker_embeddings") + os.makedirs(speaker_dir, exist_ok=True) + speaker_encoding = extract_speaker_embedding(model, args.context_audio, args.context_audio_duration) + out_path = os.path.join(speaker_dir, f"{args.speaker_name}.pt") + torch.save( + { + "speaker_encoding": speaker_encoding, + "context_audio": args.context_audio, + "embedding_dim": int(speaker_encoding.size(-1)), + "num_frames": int(speaker_encoding.size(0)), + "checkpoint": ckpt_name, + }, + out_path, + ) + logging.info(f"Saved speaker embedding '{args.speaker_name}' {tuple(speaker_encoding.shape)} to {out_path}") + + logging.info(f"Done. vLLM model directory: {args.outdir}") + + +if __name__ == "__main__": + convert(parse_args()) diff --git a/examples/tts/easymagpie_vllm_omni/easymagpie_inference_demo.ipynb b/examples/tts/easymagpie_vllm_omni/easymagpie_inference_demo.ipynb index dd7322cef37c..8cbcc17f4665 100644 --- a/examples/tts/easymagpie_vllm_omni/easymagpie_inference_demo.ipynb +++ b/examples/tts/easymagpie_vllm_omni/easymagpie_inference_demo.ipynb @@ -5,13 +5,14 @@ "id": "d5a1129d", "metadata": {}, "source": [ - "# EasyMagpieTTS — vLLM-Omni inference demo (dummy weights)\n", + "# EasyMagpieTTS — vLLM-Omni inference demo\n", "\n", "This notebook runs an end-to-end inference pass through the\n", - "[`easymagpie_vllm_omni`](./easymagpie_vllm_omni) model definition using\n", - "**dummy / random weights**, so you can exercise the full engine path\n", - "(prefill -> autoregressive decode -> audio-code extraction) without a converted\n", - "checkpoint.\n", + "[`easymagpie_vllm_omni`](./easymagpie_vllm_omni) model definition using a\n", + "**converted checkpoint directory** produced by\n", + "[`easy_magpietts_convert_to_vllm.py`](./easy_magpietts_convert_to_vllm.py)\n", + "(weights + `config.json` + text tokenizer + speaker embeddings). It exercises the\n", + "full engine path: prefill -> autoregressive decode -> audio-code extraction.\n", "\n", "It follows the same `AsyncOmni` single-stage pattern as the reference\n", "`qwen3-tts` and `eartts` demos:\n", @@ -23,19 +24,16 @@ " `context_text` itself. `prompt_token_ids = [0] * prompt_len`, sized with\n", " `EasyMagpieTTSForConditionalGeneration.estimate_prompt_len(...)`.\n", "* **decode** — each step consumes one subword id from the streaming\n", - " `additional_information.text_tokens` list; the local transformer samples all\n", - " `C * S` stacked audio codebooks for the frame.\n", + " `additional_information.text_tokens` list (the tokenized target sentence); the\n", + " local transformer samples all `C * S` stacked audio codebooks for the frame.\n", "* **output** — per-step audio codes are surfaced on\n", " `OmniOutput.multimodal_outputs[\\\"audio_codes\\\"]` (`BT x num_codebooks`), and the\n", " engine accumulates them across steps just like eartts, so we trim to the last\n", " `len(token_ids)` decoded rows.\n", "\n", - "> **Dummy weights.** We build a `config.json` sized to the real checkpoint\n", - "> (`2605_EMTTS_SmallMamba_Step150k_posttrained_epoch12.nemo`) and start the\n", - "> engine with `load_format=\\\"dummy\\\"`, so vLLM fills all parameters with random\n", - "> values. The emitted codes are therefore meaningless — this is a *smoke test*\n", - "> of the engine wiring, not a real synthesis. Point the engine at a real\n", - "> converted checkpoint (and drop `load_format`) to get audio.\n", + "> **Converted checkpoint.** Set `MODEL_DIR` below to the directory written by the\n", + "> converter. The engine reads the `config.json`, weights, and tokenizer straight\n", + "> from it — no hardcoded config, no dummy weights.\n", "\n", "> **Environment.** Run this inside the bootstrapped `vllm_omni_env` (vLLM +\n", "> vLLM-Omni + compatible torch) with the plugin installed:\n", @@ -71,7 +69,7 @@ "\n", "# Importing the model package is optional (the engine resolves the arch via the\n", "# `vllm.general_plugins` entry point installed with the package), but doing it\n", - "# here surfaces the arch dataclass we use to size the dummy prompt embedding.\n", + "# here surfaces the arch dataclass we use to read scalars from the config.\n", "from easymagpie_vllm_omni.config import EasyMagpieOmniArch\n", "\n", "print(\"torch:\", torch.__version__, \"| cuda:\", torch.cuda.is_available())" @@ -82,34 +80,30 @@ "id": "f7ff55fe", "metadata": {}, "source": [ - "## 1. Build a tiny dummy model directory\n", + "## 1. Point at the converted model directory\n", "\n", - "The engine only needs a `config.json` that (a) names the registered arch and\n", - "(b) carries the EasyMagpie + Nemotron-H scalars. Here we size everything to match\n", - "the real checkpoint\n", - "`2605_NemotronTTS_V0.2/v2/2605_EMTTS_SmallMamba_Step150k_posttrained_epoch12.nemo`\n", - "(hidden 1536, 8 codebooks × 1024, frame-stacking ×2, 3-layer local transformer).\n", + "Set `MODEL_DIR` to the directory written by\n", + "[`easy_magpietts_convert_to_vllm.py`](./easy_magpietts_convert_to_vllm.py). It\n", + "already contains everything the engine needs:\n", + "\n", + "* `config.json` — the registered arch + Nemotron-H backbone scalars + EasyMagpie\n", + " scalars (read by `EasyMagpieOmniArch.from_hf_config`),\n", + "* `model.safetensors` — the converted weights (backbone + TTS submodules + the\n", + " baked per-subword `text_embedding` table),\n", + "* the text-conditioning tokenizer (`tokenizer.json` / `tokenizer_config.json`),\n", + " loaded in-engine to tokenize the per-request `context_text`,\n", + "* `speaker_embeddings/.pt` — pre-computed speaker embeddings for reference\n", + " voices.\n", "\n", "The backbone is a **Nemotron-H** hybrid (Mamba2 + attention + MoE) decoder:\n", "`EasyMagpieTTSForConditionalGeneration` constructs vLLM's `NemotronHModel` and\n", "implements the hybrid-Mamba interfaces (`HasInnerState` / `IsHybrid` /\n", "`SupportsMambaPrefixCaching`), exactly like the EasyMagpie vLLM *sidecar*. The\n", - "`nemotron_h_config` fields (`hybrid_override_pattern`, `mamba_*`, `n_routed_experts`,\n", - "…) are copied verbatim from the checkpoint.\n", - "\n", - "The EasyMagpie-specific scalars (`embedding_dim`, `num_audio_codebooks`,\n", - "`codebook_size`, `frame_stacking_factor`, `local_transformer_*`, …) are read by\n", - "`EasyMagpieOmniArch.from_hf_config`. The phoneme branch is **enabled**\n", - "(`phoneme_stacking_factor = 1`, `phoneme_vocab_size = 2051`) to match the\n", - "checkpoint; the model self-predicts phonemes, so no phoneme stream needs to be\n", - "supplied in the prompt.\n", - "\n", - "With `load_format=\\\"dummy\\\"` (set in the stage config) vLLM never reads weight\n", - "files, so no safetensors are needed. We do save the checkpoint's\n", - "text-conditioning tokenizer (`TEXT_TOKENIZER`, the Nemotron-H tokenizer that\n", - "matches `TEXT_VOCAB`) into the model dir, since the model tokenizes the\n", - "per-request `context_text` in-engine via\n", - "`AutoTokenizer.from_pretrained(model_path)`." + "phoneme branch is enabled in the converted config; the model self-predicts\n", + "phonemes, so no phoneme stream needs to be supplied in the prompt.\n", + "\n", + "We just read the `config.json` here to surface a few scalars used for building\n", + "the prompt (`text_vocab_size`, the audio EOS id, whether a task embedding exists)." ] }, { @@ -119,106 +113,26 @@ "metadata": {}, "outputs": [], "source": [ - "# Config matching the real checkpoint:\n", - "# 2605_NemotronTTS_V0.2/v2/2605_EMTTS_SmallMamba_Step150k_posttrained_epoch12.nemo\n", - "#\n", - "# The backbone is a Nemotron-H hybrid (Mamba2 + attention + MoE) decoder, wired\n", - "# through vLLM's `NemotronHModel` by `EasyMagpieTTSForConditionalGeneration`. The\n", - "# fields below are ported verbatim from the checkpoint's `model_config.yaml`\n", - "# (the `nemotron_h_config` block + the EasyMagpie scalars). With\n", - "# `load_format=\"dummy\"` the weights are random — a realistically-sized smoke test.\n", - "#\n", - "# embedding_dim == hidden_size == audio_embedding_dim == local_transformer_hidden_dim\n", - "# (all 1536 in the checkpoint) keeps every in/out projection an Identity.\n", - "HIDDEN = 1536 # nemotron_h_config.hidden_size / embedding_dim / audio_embedding_dim\n", - "NUM_AUDIO_CODEBOOKS = 8 # vector_quantizer.num_groups\n", - "CODEBOOK_SIZE = 1024 # prod(vector_quantizer.num_levels_per_group) = 4**5\n", - "FRAME_STACKING = 2 # -> num_stacked_codebooks = NUM_AUDIO_CODEBOOKS * FRAME_STACKING = 16\n", - "PHONEME_STACKING = 1 # phoneme_stacking_factor\n", - "PHONEME_VOCAB = 2051 # IPA-BPE 2048 tokenizer + 3 special tokens\n", - "TEXT_VOCAB = 131072 # nemotron_h_config.vocab_size\n", - "# Text-conditioning tokenizer that matches the checkpoint (SmallMamba uses the\n", - "# Nemotron-H tokenizer, vocab 131072 == TEXT_VOCAB). Point this at the converted\n", - "# checkpoint dir / the checkpoint's tokenizer when running a real model.\n", - "TEXT_TOKENIZER = \"nvidia/Nemotron-H-8B-Base-8K\"\n", - "\n", - "config = {\n", - " # Resolved through the `vllm.general_plugins` entry point registered by the\n", - " # `easymagpie_vllm_omni` package -> EasyMagpieTTSForConditionalGeneration.\n", - " \"architectures\": [\"EasyMagpieTTSForConditionalGeneration\"],\n", - " # Nemotron-H backbone fields (consumed by vllm NemotronHModel) — copied\n", - " # verbatim from the checkpoint's `nemotron_h_config` block.\n", - " \"model_type\": \"nemotron_h\",\n", - " \"hidden_size\": HIDDEN,\n", - " \"num_hidden_layers\": 31,\n", - " \"vocab_size\": TEXT_VOCAB,\n", - " \"num_attention_heads\": 12,\n", - " \"num_key_value_heads\": 4,\n", - " \"attention_dropout\": 0.0,\n", - " \"attention_bias\": False,\n", - " \"max_position_embeddings\": 8192,\n", - " \"mamba_num_heads\": 64,\n", - " \"mamba_head_dim\": 24,\n", - " \"ssm_state_size\": 128,\n", - " \"conv_kernel\": 4,\n", - " \"n_groups\": 8,\n", - " \"chunk_size\": 256,\n", - " \"mamba_hidden_act\": \"silu\",\n", - " \"use_conv_bias\": True,\n", - " \"use_bias\": False,\n", - " \"intermediate_size\": 4096,\n", - " \"mlp_hidden_act\": \"silu\",\n", - " \"mlp_bias\": False,\n", - " \"n_routed_experts\": 24,\n", - " \"num_experts_per_tok\": 4,\n", - " \"moe_intermediate_size\": 768,\n", - " \"moe_shared_expert_intermediate_size\": 2048,\n", - " \"n_group\": 1,\n", - " \"topk_group\": 1,\n", - " \"routed_scaling_factor\": 2.5,\n", - " \"norm_topk_prob\": True,\n", - " # 31-char layer pattern: M=Mamba2, *=attention, E=MLP/MoE (len == num_hidden_layers).\n", - " \"hybrid_override_pattern\": \"MEMEM*EMEMEM*EMEMEMEM*EMEMEMEME\",\n", - " \"layer_norm_epsilon\": 1e-5,\n", - " \"residual_in_fp32\": False,\n", - " \"tie_word_embeddings\": False,\n", - " # bfloat16, not float32: the Nemotron-H MoE layers run vLLM's fused-MoE\n", - " # Triton kernel, whose block sizes are tuned for 16-bit. In float32 the\n", - " # kernel needs ~2x shared memory and overflows the GPU limit\n", - " # (OutOfResources: shared memory). bf16 also matches the real checkpoint.\n", - " \"torch_dtype\": \"bfloat16\",\n", - " # EasyMagpie-specific scalars (read by EasyMagpieOmniArch.from_hf_config).\n", - " \"text_vocab_size\": TEXT_VOCAB,\n", - " \"embedding_dim\": HIDDEN,\n", - " \"audio_embedding_dim\": HIDDEN,\n", - " \"num_audio_codebooks\": NUM_AUDIO_CODEBOOKS,\n", - " \"codebook_size\": CODEBOOK_SIZE,\n", - " \"frame_stacking_factor\": FRAME_STACKING,\n", - " \"phoneme_stacking_factor\": PHONEME_STACKING,\n", - " \"phoneme_vocab_size\": PHONEME_VOCAB,\n", - " \"local_transformer_n_layers\": 3,\n", - " \"local_transformer_n_heads\": 12,\n", - " \"local_transformer_hidden_dim\": HIDDEN,\n", - "}\n", - "\n", - "MODEL_DIR = Path(tempfile.mkdtemp(prefix=\"easymagpie_dummy_\"))\n", - "(MODEL_DIR / \"config.json\").write_text(json.dumps(config, indent=2))\n", + "# Directory produced by easy_magpietts_convert_to_vllm.py.\n", + "MODEL_DIR = Path(\"/home/vklimkov/workspace/emp/NeMo/examples/tts/easymagpie_vllm_omni/easymp_vllm_model\")\n", + "assert (MODEL_DIR / \"config.json\").exists(), f\"No config.json under {MODEL_DIR}; run the converter first.\"\n", "\n", - "# The model tokenizes the per-request `context_text` string in-engine via\n", - "# `AutoTokenizer.from_pretrained(model_path)` (qwen3-tts style), so the model dir\n", - "# must ship the checkpoint's text-conditioning tokenizer. We save the matching\n", - "# Nemotron-H tokenizer (TEXT_TOKENIZER) into MODEL_DIR.\n", - "from transformers import AutoTokenizer\n", + "# Read the converted config to surface a few scalars used when building the\n", + "# prompt. The engine itself loads everything from MODEL_DIR; we only peek here.\n", + "config = json.loads((MODEL_DIR / \"config.json\").read_text())\n", + "arch = EasyMagpieOmniArch.from_hf_config(type(\"Cfg\", (), config))\n", "\n", - "AutoTokenizer.from_pretrained(TEXT_TOKENIZER, trust_remote_code=True).save_pretrained(MODEL_DIR)\n", - "print(f\"Dummy model dir: {MODEL_DIR}\")\n", + "# Subword id space of the baked text-embedding table (the streaming text stream\n", + "# indexes into it). The model's text BOS/EOS/CFG-UNK ids are the last 3 rows.\n", + "TEXT_VOCAB = int(config[\"text_vocab_size\"])\n", + "TEXT_EOS_ID = TEXT_VOCAB - 2 # matches EasyMagpieTTSInferenceModel.eos_id\n", "\n", - "# Sanity-check the arch the model will derive from this config.\n", - "arch = EasyMagpieOmniArch.from_hf_config(type(\"Cfg\", (), config))\n", + "print(f\"Model dir : {MODEL_DIR}\")\n", "print(f\"embedding_dim : {arch.embedding_dim}\")\n", "print(f\"num_stacked_codebooks : {arch.num_stacked_codebooks} (C*S)\")\n", "print(f\"tokens / codebook : {arch.num_all_tokens_per_codebook} (codebook_size + specials)\")\n", - "print(f\"audio_bos / audio_eos id : {arch.audio_bos_id} / {arch.audio_eos_id}\")" + "print(f\"audio_bos / audio_eos id : {arch.audio_bos_id} / {arch.audio_eos_id}\")\n", + "print(f\"text_vocab / text_eos : {TEXT_VOCAB} / {TEXT_EOS_ID}\")" ] }, { @@ -234,14 +148,12 @@ "AR TTS model these make the runner attach the per-step `audio_codes` multimodal\n", "payload to the output (with `\\\"latent\\\"` the payload is dropped because nothing\n", "downstream consumes it, and `multimodal_output[\\\"audio_codes\\\"]` comes back\n", - "`None`). Two extra knobs make this a dummy-weights run with no external assets:\n", + "`None`).\n", "\n", - "* `load_format: \\\"dummy\\\"` — vLLM initializes random weights instead of reading a\n", - " checkpoint (so `load_weights` / `init_forbidden_mask` are skipped; the\n", - " forbidden-token mask stays all-zeros, i.e. no sampling mask — fine for a smoke\n", - " test).\n", - "* `skip_tokenizer_init: true` — we feed `prompt_token_ids` + `text_tokens`\n", - " directly, so no tokenizer files are needed.\n", + "`skip_tokenizer_init: true` — we feed `prompt_token_ids` + `text_tokens`\n", + "directly, so vLLM doesn't need its own tokenizer for the prompt (the model still\n", + "loads the bundled `AutoTokenizer` from `MODEL_DIR` in-engine to tokenize\n", + "`context_text`).\n", "\n", "`max_model_len` must cover `T_ctx` (prefill) + the number of decode steps." ] @@ -253,12 +165,12 @@ "metadata": {}, "outputs": [], "source": [ - "DECODE_STEPS = 32 # number of audio frames to decode\n", + "DECODE_STEPS = 256 # max number of audio frames to decode (trimmed at audio EOS)\n", "# Prefill length is derived at prompt-build time from the speaker embedding +\n", "# tokenized context_text (see the prompt cell); these just need to be large\n", "# enough to cover prefill + decode.\n", - "MAX_MODEL_LEN = 512\n", - "MAX_NUM_BATCHED_TOKENS = 512\n", + "MAX_MODEL_LEN = 1024\n", + "MAX_NUM_BATCHED_TOKENS = 1024\n", "\n", "stage_cfg = {\n", " \"stage_args\": [\n", @@ -293,8 +205,8 @@ " # sizes are tuned for 16-bit and overflow shared memory in fp32.\n", " \"dtype\": \"bfloat16\",\n", " \"attention_backend\": \"TRITON_ATTN\",\n", - " # --- dummy-weights smoke-test knobs ---\n", - " \"load_format\": \"dummy\",\n", + " # We feed prompt_token_ids + text_tokens directly; the model still\n", + " # loads the bundled AutoTokenizer from MODEL_DIR for context_text.\n", " \"skip_tokenizer_init\": True,\n", " },\n", " \"default_sampling_params\": {\n", @@ -321,7 +233,7 @@ " log_stats=False,\n", " stage_init_timeout=300,\n", ")\n", - "print(\"Engine ready (single stage: EasyMagpie talker, dummy weights)\")" + "print(\"Engine ready (single stage: EasyMagpie talker)\")" ] }, { @@ -335,14 +247,15 @@ "\n", "* **`speaker_embedding`** `(T_audio, embedding_dim)` — the speaker-encoded\n", " context-audio embedding (the audio branch of `prepare_context_tensors`),\n", - " loaded here from `eng_speaker_emb.pt` (as written by\n", - " `easy_magpietts_extract_speaker_encoding.py`). The model assembles the full\n", - " prefill context itself as `[task_embedding? | speaker_embedding |\n", - " context_text_embedded]`.\n", + " loaded here from `MODEL_DIR/speaker_embeddings/.pt` (written by\n", + " the converter). The model assembles the full prefill context itself as\n", + " `[task_embedding? | speaker_embedding | context_text_embedded]`.\n", "* **`context_text`** — a plain conditioning string, here `\"[EN]\"`. The model\n", " tokenizes it in-engine and embeds it through the baked `text_embedding` table.\n", - "* **`text_tokens`** `list[int]` — the streaming subword stream; decode step `k`\n", - " consumes `text_tokens[k]`. We provide one id per decode step.\n", + "* **`text_tokens`** `list[int]` — the streaming subword stream: the target\n", + " sentence tokenized with the bundled tokenizer, ending with the model's text\n", + " EOS id. Decode step `k` consumes `text_tokens[k]`; once exhausted the channel\n", + " is masked off (matching the reference `... encode(transcript) + [eos_id]`).\n", "\n", "`prompt_token_ids = [0] * prompt_len` are placeholders (the model feeds the\n", "backbone via `inputs_embeds`, never these ids). `prompt_len` must equal the\n", @@ -367,17 +280,21 @@ "from easymagpie_vllm_omni.easymagpie import EasyMagpieTTSForConditionalGeneration\n", "\n", "# Speaker-encoded context audio (audio branch of prepare_context_tensors),\n", - "# produced by easy_magpietts_extract_speaker_encoding.py.\n", - "SPEAKER_EMB_FILE = \"eng_speaker_emb.pt\"\n", - "_loaded = torch.load(SPEAKER_EMB_FILE, map_location=\"cpu\")\n", + "# pre-computed by the converter into MODEL_DIR/speaker_embeddings/.pt.\n", + "SPEAKER_NAME = \"eng\"\n", + "_loaded = torch.load(MODEL_DIR / \"speaker_embeddings\" / f\"{SPEAKER_NAME}.pt\", map_location=\"cpu\")\n", "speaker_embedding = _loaded[\"speaker_encoding\"] if isinstance(_loaded, dict) else _loaded\n", "speaker_embedding = speaker_embedding.to(torch.float32)\n", "\n", "# Plain conditioning string; the model tokenizes + embeds it in-engine.\n", "CONTEXT_TEXT = \"[EN]\"\n", "\n", - "# Same tokenizer the engine loads from MODEL_DIR — used to size the prefill\n", - "# placeholders so prompt_token_ids length matches the assembled context.\n", + "# Target sentence to synthesize.\n", + "TEXT = \"Hello, this is a test of the EasyMagpie text to speech model.\"\n", + "\n", + "# Same tokenizer the engine loads from MODEL_DIR. Used to (a) size the prefill\n", + "# placeholders so prompt_token_ids length matches the assembled context, and\n", + "# (b) tokenize the target sentence into the streaming text stream.\n", "tokenizer = AutoTokenizer.from_pretrained(MODEL_DIR, trust_remote_code=True)\n", "prompt_len = EasyMagpieTTSForConditionalGeneration.estimate_prompt_len(\n", " speaker_embedding,\n", @@ -386,8 +303,10 @@ " has_task_embedding=arch.num_task_embeddings > 0,\n", ")\n", "\n", - "# Streaming subword ids: one per decode step (step k consumes text_tokens[k]).\n", - "text_tokens = torch.randint(0, TEXT_VOCAB, (DECODE_STEPS,)).tolist()\n", + "# Streaming subword ids consumed one per decode step. Mirrors the reference\n", + "# `encode(transcript) + [eos_id]` (no BOS; HF special tokens disabled so the ids\n", + "# index the baked text_embedding table directly).\n", + "text_tokens = tokenizer.encode(TEXT, add_special_tokens=False) + [TEXT_EOS_ID]\n", "\n", "additional_information = {\n", " \"speaker_embedding\": speaker_embedding, # (T_audio, embedding_dim) tensor\n", @@ -407,15 +326,16 @@ "\n", "print(f\"speaker_embedding : {tuple(speaker_embedding.shape)}\")\n", "print(f\"context_text : {CONTEXT_TEXT!r} -> {tokenizer.encode(CONTEXT_TEXT)}\")\n", + "print(f\"text : {TEXT!r}\")\n", + "print(f\"text_tokens (len {len(text_tokens):3d}) : {text_tokens[:8]}{' ...' if len(text_tokens) > 8 else ''}\")\n", "print(f\"prompt_len (placeholders) : {prompt_len}\")\n", "print(f\"decode steps (max_tokens) : {DECODE_STEPS}\")\n", - "print(f\"text_tokens[:8] : {text_tokens[:8]}\")\n", "\n", "sampling_params = SamplingParams(\n", - " temperature=0.0,\n", + " temperature=0.0, # backbone token sampler is a no-op (audio is sampled in the local transformer)\n", " max_tokens=DECODE_STEPS,\n", " detokenize=False,\n", - " ignore_eos=True, # dummy logits never emit a meaningful EOS -> run the full budget\n", + " ignore_eos=True, # audio EOS lives in the codes, not the vLLM token stream -> run the budget + trim\n", ")" ] }, @@ -431,7 +351,8 @@ "`multimodal_output[\\\"audio_codes\\\"]` holds one row per flat-batch token over the\n", "whole run (the `T_ctx` prefill frames — codes left zero — plus one frame per\n", "decode step), so we trim to the last `len(token_ids)` rows to recover just the\n", - "decoded frames." + "decoded frames, then trim again at the audio EOS frame (the model signals\n", + "end-of-speech in the codes, not in the vLLM token stream)." ] }, { @@ -471,6 +392,17 @@ " # frames (the last len(token_ids) rows), exactly like the eartts demo.\n", " if len(token_ids) > 0:\n", " audio_codes = audio_codes[-len(token_ids):].contiguous()\n", + "\n", + " # Trim at the audio EOS: the model signals end-of-speech inside the codes\n", + " # (codebook 0 == audio_eos_id), not via the vLLM token stream.\n", + " eos_frames = (audio_codes[:, 0] == arch.audio_eos_id).nonzero(as_tuple=True)[0]\n", + " if eos_frames.numel() > 0:\n", + " eos_idx = int(eos_frames[0])\n", + " print(f\"audio EOS at frame : {eos_idx} / {audio_codes.shape[0]}\")\n", + " audio_codes = audio_codes[:eos_idx].contiguous()\n", + " else:\n", + " print(f\"no audio EOS within budget ({DECODE_STEPS} frames); using full decode\")\n", + "\n", " print(f\"audio_codes shape (decode) : {tuple(audio_codes.shape)}\")\n", " print(f\"audio_codes dtype : {audio_codes.dtype}\")\n", " print(f\"codes min / max : {int(audio_codes.min())} / {int(audio_codes.max())}\")\n", @@ -493,13 +425,124 @@ "plt.show()" ] }, + { + "cell_type": "markdown", + "id": "a32b07d5", + "metadata": {}, + "source": [ + "## 5. Decode audio codes to a waveform\n", + "\n", + "The engine emits **stacked** codebooks: `audio_codes` is `(T, C*S)` where\n", + "`C = num_audio_codebooks` and `S = frame_stacking_factor` (here `C*S = 16`).\n", + "To turn them back into a waveform we mirror what\n", + "[`EasyMagpieTTSInferenceModel.streaming_finalize`](../../../nemo/collections/tts/models/easy_magpietts_inference.py)\n", + "does at the end of inference:\n", + "\n", + "1. **load the codec** — the `.nemo` audio codec used to train the model\n", + " (`AudioCodecModel.restore_from(...)`, discriminator dropped to save memory),\n", + "2. **unstack** `(T, C*S)` -> `(1, C, T*S)` — the inverse of `stack_codes`,\n", + "3. **convert codec tokens** — this checkpoint was trained on a *regrouped* FSQ\n", + " index space (8 codebooks of size 1024) that differs from the codec's native\n", + " `GroupFiniteScalarQuantizer` (4 codebooks), so we map the model's tokens back\n", + " to the codec's space (`convert_new_to_original`) before decoding. The\n", + " `vector_quantizer` config is read straight from the source EasyMagpie `.nemo`\n", + " (config only, no weights). If the two spaces match, this step is skipped.\n", + "4. **decode** `codec_model.decode(tokens=..., tokens_len=...)` -> waveform at\n", + " `codec_model.output_sample_rate`.\n", + "\n", + "> Set `CODEC_MODEL_PATH` / `EASYMAGPIE_NEMO` to the **same** `.nemo` files passed\n", + "> to `easy_magpietts_convert_to_vllm.py` (`--codec_model_path` / `--nemo_file`).\n", + "> This step needs NeMo importable in the current environment." + ] + }, { "cell_type": "code", "execution_count": null, - "id": "3a6603b9", + "id": "aa57a573", "metadata": {}, "outputs": [], - "source": [] + "source": [ + "from hydra.utils import instantiate\n", + "from IPython.display import Audio, display\n", + "\n", + "from nemo.collections.tts.models import AudioCodecModel\n", + "from nemo.collections.tts.models.easy_magpietts_inference import EasyMagpieTTSInferenceModel\n", + "from nemo.collections.tts.modules.audio_codec_modules import VectorQuantizerIndexConverter\n", + "\n", + "# Same .nemo codec passed to easy_magpietts_convert_to_vllm.py --codec_model_path.\n", + "CODEC_MODEL_PATH = \"/home/vklimkov/workspace/emp/ckpt/easymagpietts_NEXT/25fps_spectral_codec_with_bandwidth_extension.nemo\"\n", + "\n", + "# --- load the codec once (drop the discriminator to save memory) ---\n", + "_codec_cfg = AudioCodecModel.restore_from(CODEC_MODEL_PATH, return_config=True)\n", + "if \"use_scl_loss\" in _codec_cfg:\n", + " _codec_cfg.use_scl_loss = False\n", + "codec_model = AudioCodecModel.restore_from(\n", + " CODEC_MODEL_PATH, strict=False, override_config_path=_codec_cfg\n", + ")\n", + "if hasattr(codec_model, \"discriminator\"):\n", + " del codec_model.discriminator\n", + "codec_model = codec_model.eval().to(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", + "codec_device = next(codec_model.parameters()).device\n", + "\n", + "# Source EasyMagpie .nemo (--nemo_file). Only its config is read (no weights),\n", + "# to recover the `vector_quantizer` override the model was trained with.\n", + "EASYMAGPIE_NEMO = \"/home/vklimkov/workspace/emp/ckpt/easymagpietts_NEXT/2605_NemotronTTS_V0.2/v2/2605_EMTTS_SmallMamba_Step150k_posttrained_epoch12.nemo\"\n", + "\n", + "# --- optional codec-token converter ---------------------------------------\n", + "# EasyMagpie can be trained on a *regrouped* FSQ index space (here C=8 codebooks\n", + "# of size 1024) that differs from the codec's native quantizer (this codec's\n", + "# GroupFiniteScalarQuantizer has 4 codebooks). When they differ the model's\n", + "# tokens must be mapped back to the codec's space before decoding, exactly as\n", + "# EasyMagpieTTSInferenceModel does via `_codec_converter` /\n", + "# `CodecHelper.codes_to_audio`.\n", + "_em_cfg = EasyMagpieTTSInferenceModel.restore_from(EASYMAGPIE_NEMO, return_config=True)\n", + "_vq_cfg = _em_cfg.get(\"vector_quantizer\")\n", + "if _vq_cfg is not None and instantiate(_vq_cfg).num_codebooks != codec_model.vector_quantizer.num_codebooks:\n", + " codec_converter = VectorQuantizerIndexConverter(\n", + " vector_quantizer_original=codec_model.vector_quantizer,\n", + " vector_quantizer_new=instantiate(_vq_cfg),\n", + " ).to(codec_device)\n", + "else:\n", + " codec_converter = None\n", + "print(f\"codec native codebooks : {codec_model.vector_quantizer.num_codebooks}\")\n", + "print(f\"codec token converter : {'enabled' if codec_converter is not None else 'not needed'}\")\n", + "\n", + "S = arch.frame_stacking_factor # stacking factor (sub-frames per stacked frame)\n", + "C = arch.num_stacked_codebooks // S # real codec codebooks\n", + "assert audio_codes.dim() == 2 and audio_codes.size(1) == arch.num_stacked_codebooks, (\n", + " f\"expected audio_codes (T, {arch.num_stacked_codebooks}); got {tuple(audio_codes.shape)}\"\n", + ")\n", + "\n", + "# --- unstack (T, C*S) -> (1, C, T*S): inverse of EasyMagpie stack_codes ---\n", + "stacked = audio_codes.to(codec_device, torch.long).T.unsqueeze(0) # (1, C*S, T)\n", + "T_out = stacked.size(-1)\n", + "codes = stacked.view(1, C, S, T_out).permute(0, 1, 3, 2).reshape(1, C, T_out * S) # (1, C, T*S)\n", + "codes_len = torch.tensor([codes.size(-1)], device=codec_device, dtype=torch.long)\n", + "\n", + "# Pad very short sequences (codec needs a few frames), matching _prepare_codes_for_decode.\n", + "MIN_LEN = 4\n", + "if int(codes_len.min()) < MIN_LEN:\n", + " codes = torch.nn.functional.pad(codes, (0, MIN_LEN - int(codes_len.min())), value=0)\n", + " codes_len = codes_len.clamp(min=MIN_LEN)\n", + "\n", + "# Drop any stray special tokens (BOS/EOS/MASK live at codebook_size..) so every\n", + "# index is a valid codec entry before decoding.\n", + "codes = codes.clamp_(0, arch.codebook_size - 1)\n", + "\n", + "# --- decode codes -> waveform (mirrors CodecHelper.codes_to_audio) ---\n", + "with torch.no_grad(), torch.autocast(device_type=codec_device.type, dtype=torch.float32):\n", + " if codec_converter is not None:\n", + " codes = codec_converter.convert_new_to_original(audio_tokens=codes, audio_lens=codes_len)\n", + " audio, audio_len = codec_model.decode(tokens=codes, tokens_len=codes_len)\n", + "\n", + "waveform = audio[0, : int(audio_len[0])].detach().cpu().float().numpy()\n", + "sample_rate = int(codec_model.output_sample_rate)\n", + "\n", + "print(f\"codes (unstacked) shape : {tuple(codes.shape)} (1, C={C}, T*S={codes.size(-1)})\")\n", + "print(f\"waveform samples : {waveform.shape[0]} ({waveform.shape[0] / sample_rate:.2f}s @ {sample_rate} Hz)\")\n", + "\n", + "display(Audio(waveform, rate=sample_rate))" + ] } ], "metadata": { diff --git a/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/easymagpie.py b/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/easymagpie.py index cb5e8bd346f4..fd0ff1b79f13 100644 --- a/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/easymagpie.py +++ b/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/easymagpie.py @@ -665,11 +665,14 @@ def _preprocess_decode( ) -> tuple[torch.Tensor, torch.Tensor, dict[str, Any]]: decode_offset = int(info_dict.get("ear_decode_offset", 0) or 0) - # Text channel (streaming list that grows by one per step). + # Text channel (streaming list, one subword consumed per step). Step k + # consumes text_tokens[k] (the list ends with the text eos id). Once the + # stream is exhausted the channel is masked off (adds nothing) — matching + # the reference ``text_finished`` behaviour, which stops adding text after + # EOS rather than repeating the last token. text_tokens = info_dict.get("text_tokens") - if isinstance(text_tokens, list) and text_tokens: - idx = min(decode_offset, len(text_tokens) - 1) - self._dec_text_tokens[start] = int(text_tokens[idx]) + if isinstance(text_tokens, list) and decode_offset < len(text_tokens): + self._dec_text_tokens[start] = int(text_tokens[decode_offset]) self._dec_text_mask[start] = 1 else: self._dec_text_mask[start] = 0 @@ -782,6 +785,14 @@ def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: target.data.copy_(tensor.to(target.dtype)) loaded.add(mapped) + # ``NemotronHModel.load_weights`` (the inner model) does *not* apply the + # HF->vLLM renaming that lives on the ``NemotronHForCausalLM`` wrapper, so + # raw HF names such as ``embeddings.weight`` / ``...mixer.A_log`` would not + # match the inner param names (``embed_tokens.weight`` / ``...mixer.A``). + # Apply that mapper here so the converted checkpoint can keep stock HF + # Nemotron-H names. The wrapper's ``backbone -> model`` prefix rule is a + # no-op here because we already stripped the ``decoder.`` prefix. + backbone_weights = list(NemotronHForCausalLM.hf_to_vllm_mapper.apply(backbone_weights)) backbone_loaded = self.backbone.load_weights(backbone_weights) loaded |= {f"backbone.{n}" for n in backbone_loaded} From 85be1282da9d5d4fe3010831f1e1e83163d2a344 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimkov Date: Tue, 2 Jun 2026 18:20:44 +0200 Subject: [PATCH 07/45] examples/tts/easymagpie_vllm_omni: clean up, add readme Signed-off-by: Viacheslav Klimkov --- examples/tts/easymagpie_vllm_omni/README.md | 27 +++ .../easymagpie_vllm_omni/easymagpie.py | 166 ++++++++---------- .../easymagpie_vllm_omni/local_transformer.py | 90 ++++------ 3 files changed, 134 insertions(+), 149 deletions(-) create mode 100644 examples/tts/easymagpie_vllm_omni/README.md diff --git a/examples/tts/easymagpie_vllm_omni/README.md b/examples/tts/easymagpie_vllm_omni/README.md new file mode 100644 index 000000000000..27cc34a08d34 --- /dev/null +++ b/examples/tts/easymagpie_vllm_omni/README.md @@ -0,0 +1,27 @@ +WIP model definition of EasyMP for vllm-omni. Follows footsteps of qwen3tts: +backbone and LT are compiled into a single cuda graph during uniform batch decoding, +piecewise during mixed/prefill. + +Install: +``` +pip install -e ".[all]" +pip install ninja mamba_ssm causal_conv1d --no-build-isolation +# install vllm +pip install vllm==0.21.0 vllm_omni==0.21.0rc1 +# register vllm models +pip install -e examples/tts/easymagpie_vllm_omni/ +``` + +Conver the checkpoint from +https://huggingface.co/nvidia/easymagpietts_NEXT/tree/main/2605_NemotronTTS_V0.2/v2 +``` +python examples/tts/easymagpie_vllm_omni/easy_magpietts_convert_to_vllm.py \ + --nemo_file /2605_EMTTS_SmallMamba_Step150k_posttrained_epoch12.nemo \ + --codec_model_path /25fps_spectral_codec_with_bandwidth_extension.nemo \ + --outdir examples/tts/easymagpie_vllm_omni/easymp_vllm_model \ + --context_audio english_sample.wav --speaker_name eng \ + --phoneme_tokenizer_path /bpe_ipa_tokenizer_2048_en_de_es_fr_hi_it_vi_zh_ko-KR_pt-BR_ar.json +``` + +Finally run notebook `examples/tts/easymagpie_vllm_omni/easymagpie_inference_demo.ipynb` +to predict acoustic tokens \ No newline at end of file diff --git a/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/easymagpie.py b/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/easymagpie.py index fd0ff1b79f13..c1d173a92402 100644 --- a/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/easymagpie.py +++ b/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/easymagpie.py @@ -13,53 +13,46 @@ # limitations under the License. """Inference-only EasyMagpieTTS model for vLLM-Omni. -EasyMagpieTTS is a decoder-only streaming TTS model: a text-LM backbone (the -SmallMamba checkpoint uses a Nemotron-H hybrid Mamba2 + attention + MoE decoder) -consumes a per-frame additive input embedding (text + phoneme + audio) and -emits a per-frame hidden state, from which a small autoregressive *local -transformer* samples all ``C * S`` stacked audio codebooks for that frame -(see :mod:`easymagpie_vllm_omni.local_transformer`). +EasyMagpieTTS is a decoder-only streaming TTS model. A Nemotron-H hybrid +(Mamba2 + attention + MoE) text-LM backbone consumes a per-frame additive input +embedding (text + phoneme + audio) and emits a per-frame hidden state. A small +autoregressive *local transformer* then samples all ``C * S`` stacked audio +codebooks for that frame (see :mod:`easymagpie_vllm_omni.local_transformer`). This module wires that architecture into vLLM-Omni's ``preprocess`` / ``forward`` / ``compute_logits`` / ``make_omni_output`` / -``postprocess`` contract, following the same conventions as the upstream -qwen3-tts and eartts vLLM-Omni model definitions: +``postprocess`` contract: * **Backbone** — vLLM's - :class:`~vllm.model_executor.models.nemotron_h.NemotronHModel`, reused - wholesale (hybrid Mamba2 state + KV cache + paged attention) exactly like the - EasyMagpie vLLM *sidecar*. Every step feeds the backbone via ``inputs_embeds``; - its own ``embed_tokens`` table is never consumed. Because the backbone is a - hybrid-Mamba model, the class implements vLLM's - :class:`HasInnerState` / :class:`IsHybrid` / :class:`SupportsMambaPrefixCaching` - contracts (mamba-state shape/dtype/copy helpers are delegated to - :class:`NemotronHForCausalLM`), and the SmallMamba SiLU shared-experts fix is + :class:`~vllm.model_executor.models.nemotron_h.NemotronHModel` is reused + wholesale (hybrid Mamba2 state + KV cache + paged attention). Every step feeds + the backbone via ``inputs_embeds``; its own ``embed_tokens`` table is never + consumed. Because the backbone is a hybrid-Mamba model, the class implements + vLLM's :class:`HasInnerState` / :class:`IsHybrid` / + :class:`SupportsMambaPrefixCaching` contracts (mamba-state helpers are + delegated to :class:`NemotronHForCausalLM`), and a SiLU shared-experts fix is applied at construction (see :mod:`easymagpie_vllm_omni.backbone_patches`). -* **Local transformer** — :class:`EasyMagpieCodePredictor`, a from-scratch, - CUDA-graph-capturable re-implementation that runs as a single compiled graph. -* **compute_logits** — returns trivial logits (à la eartts) so vLLM's sampler - always picks index 0; the real audio output is the codes tensor surfaced - through :meth:`make_omni_output` under the ``"audio_codes"`` key. +* **Local transformer** — :class:`EasyMagpieCodePredictor`, a + CUDA-graph-capturable implementation that runs as a single compiled graph. +* **compute_logits** — returns trivial logits so vLLM's sampler always picks + index 0; the real audio output is the codes tensor surfaced through + :meth:`make_omni_output` under the ``"audio_codes"`` key. Text is embedded via a precomputed per-subword lookup table baked at -checkpoint-conversion time (the reference char-aware subword encoder is -deterministic per subword id, so it is never run inside the engine). +checkpoint-conversion time, so the char-aware subword encoder is never run +inside the engine. Per-request I/O (via ``additional_information``): -* ``speaker_embedding`` (prefill only) — ``(T_audio, embedding_dim)`` speaker- - encoded context-audio embedding (the audio branch of the reference - ``prepare_context_tensors``), e.g. the tensor saved by - ``easy_magpietts_extract_speaker_encoding.py``. ``preprocess`` assembles the - full prefill context embedding itself as - ``[task_embedding | speaker_embedding | context_text_embedded]`` — the same - layout the reference model builds — so the caller only does the speaker-encoder - math and passes plain context text (the model tokenizes + embeds it and - prepends the per-mode service token). +* ``speaker_embedding`` (prefill only) — ``(T_audio, embedding_dim)`` + speaker-encoded context-audio embedding. ``preprocess`` assembles the full + prefill context embedding itself as + ``[task_embedding | speaker_embedding | context_text_embedded]``, so the + caller only does the speaker-encoder math and passes plain context text (the + model tokenizes + embeds it and prepends the per-mode service token). * ``context_text`` (prefill only, optional) — plain conditioning string (e.g. ``"[EN]"``); tokenized in-model with the checkpoint's text tokenizer and - embedded through the baked per-subword ``text_embedding`` table. Defaults to - ``"[NO TEXT CONTEXT]"`` when omitted. + embedded through the baked per-subword ``text_embedding`` table. * ``task_mode_id`` (prefill only, optional) — int selecting the per-mode task ("service token") embedding row; defaults to ``0``. Ignored for single-mode checkpoints (no ``task_embedding`` table). @@ -111,12 +104,10 @@ _DEFAULT_CONTEXT_TEXT = "[EN]" -# NOTE: unlike the Qwen2 backbone variant, this class is *not* wrapped in -# ``@support_torch_compile``. The Nemotron-H backbone is a hybrid-Mamba model -# that manages its own ``torch.compile`` / CUDA-graph capture internally (as -# does :class:`EasyMagpieCodePredictor`), so the outer ``forward`` runs eagerly -# and dispatches into the two self-compiled subgraphs — matching the EasyMagpie -# vLLM sidecar (``EasyMagpieSmallMamba``). +# This class is not wrapped in ``@support_torch_compile``: the Nemotron-H +# backbone and :class:`EasyMagpieCodePredictor` each manage their own +# ``torch.compile`` / CUDA-graph capture internally, so the outer ``forward`` +# runs eagerly and dispatches into the two self-compiled subgraphs. class EasyMagpieTTSForConditionalGeneration( nn.Module, HasInnerState, @@ -131,8 +122,8 @@ class EasyMagpieTTSForConditionalGeneration( ``OmniGPUModelRunner``. """ - # Hybrid-Mamba bookkeeping (delegated to vLLM's NemotronH causal-LM, exactly - # like the EasyMagpie sidecar). vLLM expects these as class attributes. + # Hybrid-Mamba bookkeeping (delegated to vLLM's NemotronH causal-LM). vLLM + # expects these as class attributes. get_mamba_state_dtype_from_config = NemotronHForCausalLM.get_mamba_state_dtype_from_config get_mamba_state_shape_from_config = NemotronHForCausalLM.get_mamba_state_shape_from_config get_mamba_state_copy_func = NemotronHForCausalLM.get_mamba_state_copy_func @@ -167,9 +158,9 @@ def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: vllm_config=vllm_config, prefix=maybe_prefix(prefix, "backbone"), ) - # SmallMamba was trained with mlp_hidden_act=silu but vLLM's NemotronHMLP - # hard-codes ReLU² in shared_experts. Restore SiLU (no-op when the - # backbone has no MoE layers). + # The checkpoint was trained with mlp_hidden_act=silu but vLLM's + # NemotronHMLP hard-codes ReLU² in shared_experts. Restore SiLU (no-op + # when the backbone has no MoE layers). patch_silu_shared_experts(self.backbone) # ── Local transformer (its own compile group / CUDA graph) ────── @@ -180,26 +171,21 @@ def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: ) # ── Text + phoneme embedding heads ────────────────────────────── - # Precomputed per-subword text embedding. The reference model embeds - # text with a char-aware subword (CAS) encoder + the decoder's subword - # table; both are deterministic per subword id, so the checkpoint - # converter bakes their combined result into this single lookup table - # (one row per subword id). It is fed additively on every decode step; - # the CAS encoder is never run inside the engine. + # Precomputed per-subword text embedding (one row per subword id), baked + # at conversion time and fed additively on every decode step. text_vocab_size = int(getattr(hf_config, "text_vocab_size", getattr(hf_config, "vocab_size", 0))) self.text_embedding = nn.Embedding(text_vocab_size, self.embedding_dim) - # Task ("service token") embedding — a single learned per-mode row the - # reference model prepends to the prefill context when trained with >1 - # mode. Built only when the checkpoint carries one; otherwise ``None``. + # Task ("service token") embedding — a single learned per-mode row + # prepended to the prefill context for multi-mode checkpoints. Built only + # when the checkpoint carries one; otherwise ``None``. self.num_task_embeddings = int(arch.num_task_embeddings) if self.num_task_embeddings > 0: self.task_embedding = nn.Embedding(self.num_task_embeddings, self.embedding_dim) else: self.task_embedding = None - # Context-text tokenizer, loaded lazily from the model directory (same - # ``AutoTokenizer.from_pretrained(model_path)`` pattern as qwen3-tts). It + # Context-text tokenizer, loaded lazily from the model directory. It # turns the per-request ``context_text`` string (e.g. ``"[EN]"``) into the # subword ids that the baked ``text_embedding`` table consumes — so the # caller passes plain text, never pre-tokenized ids. @@ -295,8 +281,6 @@ def _select_query_layout(attn_metadata): def _get_decode_idxs(self): """Return ``(decode_token_indices, num_requests)`` for code-predictor dispatch. - Mirrors the qwen3-tts / eartts pattern: - * ``(None, 0)`` → run the local transformer on every token (profile / dummy run with no ``attn_metadata``, or a decode-only batch where ``max_query_len == 1``), so the captured CUDA graph covers every @@ -434,9 +418,9 @@ def _predict_phonemes(self, hidden_states: torch.Tensor, idx) -> None: def compute_logits(self, hidden_states, sampling_metadata: Any = None) -> Optional[torch.Tensor]: """Return zero logits so vLLM's sampler always picks index 0. - The width is taken from ``hf_config.vocab_size`` so the sampler's - working buffers match. The sampled id is irrelevant — audio is surfaced - via :meth:`make_omni_output`. + The width is taken from ``hf_config.vocab_size`` so the sampler's working + buffers match. The sampled id is irrelevant — audio is surfaced via + :meth:`make_omni_output`. """ if isinstance(hidden_states, OmniOutput): hidden_states = hidden_states.text_hidden_states @@ -520,7 +504,7 @@ def _preprocess_prefill( ) -> tuple[torch.Tensor, torch.Tensor, dict[str, Any]]: prefill_embeds = self._build_prefill_embeds(device, info_dict) - offset = int(info_dict.get("ear_prefill_offset", 0) or 0) + offset = int(info_dict.get("prefill_offset", 0) or 0) total = int(prefill_embeds.shape[0]) s = max(0, min(offset, total)) e = max(0, min(offset + span_len, total)) @@ -535,8 +519,8 @@ def _preprocess_prefill( take = torch.cat([take, pad_rows], dim=0) info_update = { - "ear_prefill_offset": offset + span_len, - "ear_decode_offset": 0, + "prefill_offset": offset + span_len, + "decode_offset": 0, } input_ids_out = torch.full_like(input_ids, _DUMMY_TOKEN_ID) return input_ids_out, take, info_update @@ -546,23 +530,17 @@ def _build_prefill_embeds( device: torch.device, info_dict: dict[str, Any], ) -> torch.Tensor: - """Assemble the full ``(T_ctx, embedding_dim)`` prefill context embedding. - - Reproduces the prefill assembly from the reference - ``prepare_context_tensors``:: + """Assemble the full ``(T_ctx, embedding_dim)`` prefill context embedding:: [task_embedding | speaker_embedding | context_text_embedded] from the per-request inputs: - * ``speaker_embedding`` — the speaker-encoded context-audio embedding - (e.g. produced by ``easy_magpietts_extract_speaker_encoding.py``), + * ``speaker_embedding`` — the speaker-encoded context-audio embedding, required as a 2-D ``(T_audio, embedding_dim)`` tensor. * ``context_text`` — a plain string (e.g. ``"[EN]"``); tokenized in-model (see :meth:`_encode_context_text`) and embedded through the baked - per-subword ``text_embedding`` table (which already folds in the CAS - encoder, matching the default ``disable_cas_for_context_text=False`` - training). Defaults to ``"[NO TEXT CONTEXT]"`` when omitted. + per-subword ``text_embedding`` table. * ``task_mode_id`` — selects the per-mode task ("service token") embedding row; prepended only when the checkpoint has a task table. @@ -602,9 +580,9 @@ def _build_prefill_embeds( def _get_text_tokenizer(self): """Lazily load the context-text tokenizer from the model directory. - Mirrors qwen3-tts: the converted checkpoint ships a HuggingFace - ``AutoTokenizer`` (the model's text-conditioning tokenizer) alongside its - weights, so we load it on first use from ``model_path``. + The converted checkpoint ships a HuggingFace ``AutoTokenizer`` (the + model's text-conditioning tokenizer) alongside its weights, so we load it + on first use from ``model_path``. """ if self._text_tokenizer is None: from transformers import AutoTokenizer @@ -613,12 +591,11 @@ def _get_text_tokenizer(self): return self._text_tokenizer def _encode_context_text(self, context_text: str, device: torch.device) -> torch.Tensor: - """Tokenize ``context_text`` to subword ids (matching the reference encode path). + """Tokenize ``context_text`` to subword ids. - The reference ``AggregatedTTSTokenizer.encode`` calls the underlying - HF tokenizer's ``encode`` (default ``add_special_tokens``) for the - text-conditioning tokenizer, which sits at offset 0 in the aggregate, so - its raw ids index the baked ``text_embedding`` table directly. + The text-conditioning tokenizer sits at offset 0 in the model's + tokenizer aggregate, so its raw ids index the baked ``text_embedding`` + table directly. """ tok = self._get_text_tokenizer() ids = tok.encode(context_text) @@ -632,8 +609,7 @@ def estimate_prompt_len( context_text: str = _DEFAULT_CONTEXT_TEXT, has_task_embedding: bool = False, ) -> int: - """Length-only mirror of :meth:`_build_prefill_embeds` (à la qwen3-tts's - ``estimate_prompt_len_from_additional_information``). + """Length-only mirror of :meth:`_build_prefill_embeds`. The engine assembles the prefill context as ``[task_embedding? | speaker_embedding | context_text_embedded]``, so the @@ -663,13 +639,12 @@ def _preprocess_decode( device: torch.device, info_dict: dict[str, Any], ) -> tuple[torch.Tensor, torch.Tensor, dict[str, Any]]: - decode_offset = int(info_dict.get("ear_decode_offset", 0) or 0) + decode_offset = int(info_dict.get("decode_offset", 0) or 0) # Text channel (streaming list, one subword consumed per step). Step k # consumes text_tokens[k] (the list ends with the text eos id). Once the - # stream is exhausted the channel is masked off (adds nothing) — matching - # the reference ``text_finished`` behaviour, which stops adding text after - # EOS rather than repeating the last token. + # stream is exhausted the channel is masked off (adds nothing) rather than + # repeating the last token. text_tokens = info_dict.get("text_tokens") if isinstance(text_tokens, list) and decode_offset < len(text_tokens): self._dec_text_tokens[start] = int(text_tokens[decode_offset]) @@ -699,7 +674,7 @@ def _preprocess_decode( self._dec_audio_valid[start] = 1 inputs_embeds_out = torch.zeros((1, self.embedding_dim), device=device, dtype=self._combined_embeddings.dtype) - info_update = {"ear_decode_offset": decode_offset + 1} + info_update = {"decode_offset": decode_offset + 1} return input_ids, inputs_embeds_out, info_update def postprocess(self, hidden_states: torch.Tensor, multimodal_outputs: Optional[dict[str, Any]] = None, **_: Any): @@ -722,7 +697,7 @@ def postprocess(self, hidden_states: torch.Tensor, multimodal_outputs: Optional[ # weight loading # ------------------------------------------------------------------ - # Checkpoint prefixes (reference EasyMagpieTTS state dict) → in-model paths. + # Checkpoint prefixes (EasyMagpieTTS state dict) → in-model paths. # ``decoder.*`` is fed to the vLLM backbone loader separately (it understands # HF Nemotron-H naming + Mamba/MoE packing). The TTS submodules are copied # manually. @@ -749,13 +724,12 @@ def _remap_tts_key(self, name: str) -> Optional[str]: def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: """Load backbone (Nemotron-H) + TTS submodule weights from a converted checkpoint. - The converted checkpoint is expected to use the reference EasyMagpieTTS - key layout: the backbone under ``decoder.*`` (HF Nemotron-H names) and - the TTS submodules at top level (``audio_embeddings.*``, - ``local_transformer.*``, ``phoneme_*``, ``text_embedding.*``, projection - heads). Backbone weights are routed to :meth:`NemotronHModel.load_weights` - (which handles HF naming + Mamba/MoE packing); TTS weights are copied - directly by name. + The converted checkpoint carries the backbone under ``decoder.*`` (HF + Nemotron-H names) and the TTS submodules at top level + (``audio_embeddings.*``, ``local_transformer.*``, ``phoneme_*``, + ``text_embedding.*``, projection heads). Backbone weights are routed to + :meth:`NemotronHModel.load_weights` (which handles HF naming + Mamba/MoE + packing); TTS weights are copied directly by name. """ own_params = dict(self.named_parameters()) loaded: set[str] = set() diff --git a/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/local_transformer.py b/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/local_transformer.py index b48715604530..aab5fe2f224b 100644 --- a/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/local_transformer.py +++ b/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/local_transformer.py @@ -11,29 +11,24 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""From-scratch autoregressive local transformer for EasyMagpieTTS on vLLM-Omni. - -The reference EasyMagpieTTS model predicts the ``C * S`` stacked audio -codebooks of one frame *autoregressively* with a small causal transformer -(``nemo.collections.tts.modules.transformer_2501.Transformer``) conditioned on -the backbone's per-frame hidden state. The reference implementation re-creates -fresh tensors and (optionally) a KV cache on every codebook step, which is -incompatible with CUDA-graph replay. - -This module re-implements that local transformer from scratch so it can run as -a single compiled CUDA graph: - -* :class:`EasyMagpieLocalTransformer` mirrors the ``transformer_2501`` - layer/weight layout **exactly** (so a stock checkpoint loads 1:1) but uses - ``scaled_dot_product_attention`` and drops the KV cache / padding-mask - plumbing. It is decorated with ``@support_torch_compile`` so vLLM captures - one CUDA graph for the fixed ``(num_tokens, num_stacked_codebooks, hidden)`` - input shape. +"""Autoregressive local transformer for EasyMagpieTTS on vLLM-Omni. + +EasyMagpieTTS predicts the ``C * S`` stacked audio codebooks of one frame +*autoregressively* with a small causal transformer conditioned on the backbone's +per-frame hidden state. This module implements that local transformer so it can +run as a single compiled CUDA graph: + +* :class:`EasyMagpieLocalTransformer` is a causal transformer stack with + learnable positional embeddings, using ``scaled_dot_product_attention`` and no + KV cache. It is decorated with ``@support_torch_compile`` so vLLM captures one + CUDA graph for the fixed ``(num_tokens, num_stacked_codebooks, hidden)`` input + shape. Its layer/weight layout matches the training checkpoint so weights load + 1:1. * :class:`EasyMagpieCodePredictor` owns the persistent, address-stable scratch buffers and runs the per-frame autoregressive loop, re-invoking the compiled - transformer once per codebook over the **same** buffer (matching the - qwen3-tts code-predictor trick: replaying one fixed-shape graph N times is - faster and simpler than capturing N separate graphs). + transformer once per codebook over the **same** buffer (replaying one + fixed-shape graph N times is faster and simpler than capturing N separate + graphs). All sampling is CUDA-graph safe (Gumbel-max + ``topk`` + ``masked_fill`` only; no host syncs, no ``multinomial`` on possibly-degenerate warmup data). @@ -96,12 +91,11 @@ def sample_codebook( class EasyMagpieLTSelfAttention(nn.Module): - """Causal self-attention matching ``transformer_2501.SelfAttention`` weights. + """Causal self-attention. - Same projections (``qkv_net`` fused QKV without bias, ``o_net`` without - bias) and the same ``d_head ** -0.5`` scaling, but computed with - ``scaled_dot_product_attention`` and an ``is_causal=True`` flag instead of - the materialised causal-mask buffer + naive softmax. No KV cache: the + Fused QKV projection (``qkv_net``) and output projection (``o_net``), both + bias-free, with ``d_head ** -0.5`` scaling computed via + ``scaled_dot_product_attention`` with ``is_causal=True``. No KV cache: the autoregressive loop re-runs the full (short, fixed-length) sequence each step, which is what makes the whole thing CUDA-graph capturable. """ @@ -131,19 +125,16 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: class EasyMagpieLTFeedForward(nn.Module): - """Positionwise FFN matching ``transformer_2501.PositionwiseConvFF`` weights. + """Positionwise feed-forward network. - The reference uses ``Conv1d(kernel_size=1)`` layers named ``proj.conv`` and - ``o_net.conv`` (no bias). A kernel-1 conv is a plain linear over the channel - dim, so we keep the exact ``Conv1d`` submodule names — the checkpoint loads - 1:1 — and apply them with a single transpose, GELU(tanh) in between. + Uses ``Conv1d(kernel_size=1)`` layers named ``proj.conv`` and ``o_net.conv`` + (no bias). A kernel-1 conv is a plain linear over the channel dim, applied + with a single transpose and GELU(tanh) in between. The ``Conv1d`` submodule + names match the training checkpoint so weights load 1:1. """ def __init__(self, d_model: int, d_ffn: int) -> None: super().__init__() - # Wrap the Conv1d in a tiny container so the parameter path is - # ``proj.conv.weight`` / ``o_net.conv.weight`` exactly as in the - # reference ``ConvolutionLayer``. self.proj = _Conv1dWrapper(d_model, d_ffn) self.o_net = _Conv1dWrapper(d_ffn, d_model) self.act = nn.GELU(approximate="tanh") @@ -170,10 +161,8 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: class EasyMagpieLTLayer(nn.Module): """One pre-norm transformer layer (self-attn + FFN), bias-free LayerNorms. - Residual structure matches ``transformer_2501.TransformerLayer`` with an - all-ones ``x_mask`` (inference): ``x = x + attn(norm_self(x))`` then - ``x = x + ff(norm_pos_ff(x))``. The ``x * x_mask`` multiplications are - identities when nothing is padded, so they are dropped. + Residual structure: ``x = x + attn(norm_self(x))`` then + ``x = x + ff(norm_pos_ff(x))``. """ def __init__(self, d_model: int, d_ffn: int, n_heads: int) -> None: @@ -202,9 +191,8 @@ class EasyMagpieLocalTransformer(nn.Module): Decorated with ``@support_torch_compile`` so vLLM captures a single CUDA graph for the fixed ``(num_tokens, num_stacked_codebooks, d_model)`` input - shape. Weight layout mirrors ``transformer_2501.Transformer``: - ``position_embeddings`` (learnable), ``layers.{i}.*`` and a no-op - ``norm_out`` (``apply_norm_out=False`` in the reference, hence ``Identity``). + shape. Holds learnable ``position_embeddings``, the stacked ``layers.{i}.*`` + and a no-op ``norm_out``. """ def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: @@ -214,15 +202,13 @@ def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: n_heads = arch.local_transformer_n_heads n_layers = arch.local_transformer_n_layers d_ffn = d_model * 4 - # +2 matches the reference ``max_length_causal_mask`` head-room - # (``num_stacked_codebooks + 2``). + # +2 of head-room over ``num_stacked_codebooks`` for the positional table. max_len = arch.num_stacked_codebooks + 2 self.position_embeddings = nn.Embedding(max_len, d_model) self.layers = nn.ModuleList( [EasyMagpieLTLayer(d_model, d_ffn, n_heads) for _ in range(n_layers)] ) - # apply_norm_out=False in the reference config -> no parameters. self.norm_out = nn.Identity() def forward(self, inputs_embeds: torch.Tensor) -> torch.Tensor: @@ -267,8 +253,7 @@ def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: lt_hidden = arch.local_transformer_hidden_dim # Per-codebook audio token embeddings (shared with the outer model's - # decode-step input-embedding assembly). Names match the reference - # checkpoint's ``audio_embeddings.{i}``. + # decode-step input-embedding assembly). self.audio_embeddings = nn.ModuleList( [nn.Embedding(self.num_tokens_per_codebook, self.audio_embedding_dim) for _ in range(self.num_codebooks)] ) @@ -321,9 +306,9 @@ def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: def init_forbidden_mask(self) -> None: """Forbid all trailing special tokens except audio EOS. - Mirrors ``SpecialAudioToken.get_forbidden_tokens`` — everything in the - special-token block above ``codebook_size`` is blocked at sampling - time, except ``audio_eos`` which must remain reachable to terminate. + Everything in the special-token block above ``codebook_size`` is blocked + at sampling time, except ``audio_eos`` which must remain reachable to + terminate. """ mask = torch.zeros(self.num_tokens_per_codebook, dtype=torch.bool, device=self.forbidden_mask.device) mask[self.arch.codebook_size :] = True @@ -339,10 +324,9 @@ def embed_codebook(self, codebook_idx: int, codes: torch.Tensor) -> torch.Tensor def embed_audio_frame(self, codes: torch.Tensor) -> torch.Tensor: """Embed a full frame of stacked codes into the backbone embedding space. - Averages per-codebook embeddings then applies ``audio_in_projection``, - matching the reference ``embed_audio_tokens`` (which sums and divides by - the number of codebooks). Used by the outer model to build the decode - input embedding from the previous frame's codes. + Averages the per-codebook embeddings then applies ``audio_in_projection``. + Used by the outer model to build the decode input embedding from the + previous frame's codes. Args: codes: ``[num_tokens, num_codebooks]`` int64 codes. From f984ee1850ade1152227dc51376726338d6f251c Mon Sep 17 00:00:00 2001 From: Viacheslav Klimkov Date: Tue, 2 Jun 2026 18:20:44 +0200 Subject: [PATCH 08/45] examples/tts/easymagpie_vllm_omni: implement delay and proper phoneme prediction processing Signed-off-by: Viacheslav Klimkov --- .../easy_magpietts_convert_to_vllm.py | 28 ++++ .../easymagpie_inference_demo.ipynb | 26 +++- .../easymagpie_vllm_omni/config.py | 38 +++++ .../easymagpie_vllm_omni/easymagpie.py | 131 +++++++++++++++--- 4 files changed, 198 insertions(+), 25 deletions(-) diff --git a/examples/tts/easymagpie_vllm_omni/easy_magpietts_convert_to_vllm.py b/examples/tts/easymagpie_vllm_omni/easy_magpietts_convert_to_vllm.py index 664a243d7415..4cb99a08baee 100644 --- a/examples/tts/easymagpie_vllm_omni/easy_magpietts_convert_to_vllm.py +++ b/examples/tts/easymagpie_vllm_omni/easy_magpietts_convert_to_vllm.py @@ -306,6 +306,34 @@ def build_config(model, vocab_size: int, torch_dtype: str) -> dict: has_phoneme = getattr(model, "phoneme_tokenizer", None) is not None config["phoneme_stacking_factor"] = int(getattr(model, "phoneme_stacking_factor", 0)) if has_phoneme else 0 config["phoneme_vocab_size"] = int(getattr(model, "phoneme_vocab_size", 0)) if has_phoneme else 0 + if has_phoneme: + # Phoneme special-token ids + the confidence→UNK replacement threshold, + # consumed by the in-engine phoneme stream (BOS seeding, EOS-stop, UNK). + config["phoneme_bos_id"] = int(model.phoneme_tokenizer.bos_token_id) + config["phoneme_eos_id"] = int(model.phoneme_tokenizer.eos_token_id) + unk_id = getattr(model.phoneme_tokenizer, "unk_token_id", None) + if unk_id is not None: + config["phoneme_unk_id"] = int(unk_id) + config["phoneme_confidence_unk_threshold"] = float(getattr(model, "phoneme_confidence_unk_threshold", 0.0)) + + # ── Streaming delays from the default inference mode ── + # The reference offsets the text/phoneme/audio streams by these per-mode + # delays; the vLLM model reproduces them in its decode step. A 0/0 (or "full") + # mode runs the three streams in lock-step. + default_mode = model.mode_name_to_mode.get(model.default_inference_mode) + if default_mode is not None: + if default_mode.text_input_mode != "streaming": + logging.warning( + "Converting a checkpoint whose default inference mode is " + f"'{default_mode.text_input_mode}' (not 'streaming'); the vLLM model only " + "implements the streaming-mode delay semantics (audio starts after " + "`streaming_speech_delay` text tokens)." + ) + config["streaming_phonemes_delay"] = int(default_mode.streaming_phonemes_delay) + config["streaming_speech_delay"] = int(default_mode.streaming_speech_delay) + else: + config["streaming_phonemes_delay"] = 0 + config["streaming_speech_delay"] = 0 config["num_task_embeddings"] = len(model.training_modes) if model.task_embedding is not None else 0 diff --git a/examples/tts/easymagpie_vllm_omni/easymagpie_inference_demo.ipynb b/examples/tts/easymagpie_vllm_omni/easymagpie_inference_demo.ipynb index 8cbcc17f4665..ebab7bc05532 100644 --- a/examples/tts/easymagpie_vllm_omni/easymagpie_inference_demo.ipynb +++ b/examples/tts/easymagpie_vllm_omni/easymagpie_inference_demo.ipynb @@ -192,18 +192,20 @@ " \"model_arch\": \"EasyMagpieTTSForConditionalGeneration\",\n", " \"worker_type\": \"ar\",\n", " \"scheduler_cls\": \"vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler\",\n", - " #\"enforce_eager\": True, # dummy run: skip CUDA-graph capture for a faster start\n", + " \"enforce_eager\": True, # dummy run: skip CUDA-graph capture for a faster start\n", " \"trust_remote_code\": True,\n", " \"async_scheduling\": True,\n", " \"enable_prefix_caching\": False,\n", " \"engine_output_type\": \"audio\",\n", - " \"gpu_memory_utilization\": 0.6,\n", + " \"gpu_memory_utilization\": 0.8,\n", " \"distributed_executor_backend\": \"uni\",\n", " \"max_num_batched_tokens\": MAX_NUM_BATCHED_TOKENS,\n", " \"max_model_len\": MAX_MODEL_LEN,\n", " # bf16 (not fp32): the Nemotron-H fused-MoE Triton kernel's block\n", " # sizes are tuned for 16-bit and overflow shared memory in fp32.\n", - " \"dtype\": \"bfloat16\",\n", + " #\"dtype\": \"bfloat16\",\n", + " \"dtype\": \"float16\",\n", + " \"mamba_ssm_cache_dtype\": \"float32\",\n", " \"attention_backend\": \"TRITON_ATTN\",\n", " # We feed prompt_token_ids + text_tokens directly; the model still\n", " # loads the bundled AutoTokenizer from MODEL_DIR for context_text.\n", @@ -308,10 +310,19 @@ "# index the baked text_embedding table directly).\n", "text_tokens = tokenizer.encode(TEXT, add_special_tokens=False) + [TEXT_EOS_ID]\n", "\n", + "# Audio (local-transformer) sampling params. vLLM's SamplingParams.temperature\n", + "# drives only the dummy backbone token sampler, so the *audio* temperature/top-k\n", + "# are forwarded via additional_information. temperature=0.0 == argmax\n", + "# (deterministic; matches the torch reference run with --temperature 0.0 --no_cfg).\n", + "LT_TEMPERATURE = 0.0\n", + "LT_TOPK = 80\n", + "\n", "additional_information = {\n", " \"speaker_embedding\": speaker_embedding, # (T_audio, embedding_dim) tensor\n", " \"context_text\": CONTEXT_TEXT, # plain string, tokenized in-model\n", " \"text_tokens\": text_tokens, # list[int], grows by one per step\n", + " \"temperature\": LT_TEMPERATURE, # audio sampling temperature (local transformer)\n", + " \"top_k\": LT_TOPK, # audio sampling top-k (local transformer)\n", "}\n", "\n", "prompt = {\n", @@ -393,6 +404,15 @@ " if len(token_ids) > 0:\n", " audio_codes = audio_codes[-len(token_ids):].contiguous()\n", "\n", + " # Drop the leading streaming_speech_delay warm-up frames. With the streaming\n", + " # delay the audio stream only opens at decode step == speech_delay, so the\n", + " # first speech_delay decoded frames carry no real audio (the audio channel was\n", + " # masked off while the model consumed lookahead text/phonemes).\n", + " speech_delay = int(getattr(arch, \"streaming_speech_delay\", 0) or 0)\n", + " if speech_delay > 0:\n", + " print(f\"dropping {speech_delay} leading speech-delay warm-up frames\")\n", + " audio_codes = audio_codes[speech_delay:].contiguous()\n", + "\n", " # Trim at the audio EOS: the model signals end-of-speech inside the codes\n", " # (codebook 0 == audio_eos_id), not via the vLLM token stream.\n", " eos_frames = (audio_codes[:, 0] == arch.audio_eos_id).nonzero(as_tuple=True)[0]\n", diff --git a/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/config.py b/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/config.py index 1b086ec3e562..b6b76e97be0f 100644 --- a/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/config.py +++ b/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/config.py @@ -72,6 +72,23 @@ class EasyMagpieOmniArch: phoneme_stacking_factor: int = 1 phoneme_vocab_size: int = 2051 + # ── Streaming delays (per the checkpoint's default inference mode) ── + # The text/phoneme/audio streams are temporally offset: at decode step ``k`` + # the text channel consumes ``text_tokens[k]``, the phoneme channel starts at + # ``k == streaming_phonemes_delay`` (seeded with phoneme BOS), and the audio + # channel starts at ``k == streaming_speech_delay`` (seeded with audio BOS). + # Both default to 0 (lock-step), which reproduces a non-delayed / "full" mode. + streaming_phonemes_delay: int = 0 + streaming_speech_delay: int = 0 + + # Phoneme special-token ids (into the per-stack ``phoneme_embeddings`` table) + # and the confidence→UNK replacement threshold. ``None`` falls back to the + # IPABPETokenizer convention (bos/eos/unk = vocab-3/-2/-1). + phoneme_bos_id: int | None = None + phoneme_eos_id: int | None = None + phoneme_unk_id: int | None = None + phoneme_confidence_unk_threshold: float = 0.0 + # Number of multi-mode task ("service token") embeddings. The reference model # prepends a single learned per-mode embedding to the prefill context when # trained with >1 mode (``cfg.training_modes``); 0 disables it (single-mode @@ -122,6 +139,21 @@ def mask_token_id(self) -> int: return self.forced_mask_token_id return self.codebook_size + SPECIAL_AUDIO_MASK + @property + def resolved_phoneme_bos_id(self) -> int: + """Phoneme BOS id, falling back to the IPABPETokenizer convention (vocab-3).""" + return self.phoneme_bos_id if self.phoneme_bos_id is not None else self.phoneme_vocab_size - 3 + + @property + def resolved_phoneme_eos_id(self) -> int: + """Phoneme EOS id, falling back to the IPABPETokenizer convention (vocab-2).""" + return self.phoneme_eos_id if self.phoneme_eos_id is not None else self.phoneme_vocab_size - 2 + + @property + def resolved_phoneme_unk_id(self) -> int: + """Phoneme UNK id, falling back to the IPABPETokenizer convention (vocab-1).""" + return self.phoneme_unk_id if self.phoneme_unk_id is not None else self.phoneme_vocab_size - 1 + @classmethod def from_hf_config(cls, hf_config: Any) -> "EasyMagpieOmniArch": """Build an arch description from a vLLM ``hf_config``. @@ -142,6 +174,12 @@ def from_hf_config(cls, hf_config: Any) -> "EasyMagpieOmniArch": "frame_stacking_factor", "phoneme_stacking_factor", "phoneme_vocab_size", + "streaming_phonemes_delay", + "streaming_speech_delay", + "phoneme_bos_id", + "phoneme_eos_id", + "phoneme_unk_id", + "phoneme_confidence_unk_threshold", "num_task_embeddings", "local_transformer_n_layers", "local_transformer_n_heads", diff --git a/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/easymagpie.py b/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/easymagpie.py index c1d173a92402..8f01eb886bb8 100644 --- a/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/easymagpie.py +++ b/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/easymagpie.py @@ -62,8 +62,19 @@ * ``text_tokens`` — Python ``list[int]`` of subword ids that grows by one per decode step; step ``k`` consumes ``text_tokens[k]`` (embedded through the precomputed per-subword table). -* ``phoneme_tokens`` (optional) — same streaming-list contract for the phoneme - channel; if omitted the phoneme branch is skipped. +* ``temperature`` / ``top_k`` (prefill only, optional) — audio sampling params + for the local transformer. vLLM's ``SamplingParams.temperature`` drives only + the dummy backbone token sampler, so the *audio* temperature/top-k are passed + here and applied to the code predictor (defaults: ``0.7`` / ``80``). + +Streaming delays: the text, phoneme and audio streams are temporally offset by +the checkpoint's ``streaming_phonemes_delay`` / ``streaming_speech_delay`` (baked +into ``config.json`` by the converter from the default inference mode). The text +stream runs from decode step 0; the phoneme stream opens at step +``phonemes_delay`` (seeded with phoneme BOS) and the audio stream at step +``speech_delay`` (seeded with audio BOS). The leading ``speech_delay`` decoded +frames are warm-up only and must be dropped by the caller. Delays of 0/0 +reproduce a lock-step / non-delayed model. """ from __future__ import annotations @@ -191,6 +202,11 @@ def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: # caller passes plain text, never pre-tokenized ids. self._text_tokenizer: Any = None + # ── Streaming delays (text leads phoneme by ``phonemes_delay`` and audio + # by ``speech_delay`` decode steps; 0/0 == lock-step). ── + self.phonemes_delay = int(getattr(arch, "streaming_phonemes_delay", 0) or 0) + self.speech_delay = int(getattr(arch, "streaming_speech_delay", 0) or 0) + # Phoneme channel (optional — only built when the checkpoint has one). self.has_phoneme = arch.phoneme_vocab_size > 0 and arch.phoneme_stacking_factor > 0 if self.has_phoneme: @@ -200,6 +216,11 @@ def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: self.phoneme_final_proj = nn.Linear( self.hidden_dim, arch.phoneme_vocab_size * arch.phoneme_stacking_factor ) + # Phoneme special-token ids + confidence→UNK replacement threshold. + self.phoneme_bos_id = int(arch.resolved_phoneme_bos_id) + self.phoneme_eos_id = int(arch.resolved_phoneme_eos_id) + self.phoneme_unk_id = int(arch.resolved_phoneme_unk_id) + self.phoneme_confidence_unk_threshold = float(arch.phoneme_confidence_unk_threshold) # ── Persistent, address-stable scratch buffers ───────────────── max_num_tokens = vllm_config.scheduler_config.max_num_batched_tokens @@ -401,14 +422,34 @@ def _assemble_decode_embeddings(self, combined: torch.Tensor, idx) -> None: @torch.no_grad() def _predict_phonemes(self, hidden_states: torch.Tensor, idx) -> None: - """Argmax the phoneme head and stash the prediction for the next step.""" + """Argmax the phoneme head (with confidence→UNK replacement) and stash it. + + The UNK replacement mirrors the reference: when the max phoneme + probability of any stacked channel falls below + ``phoneme_confidence_unk_threshold`` (and the step is not an EOS step), + the whole step is replaced with the UNK id to curb error propagation. + + This is done here — not in ``preprocess``/``postprocess`` — because this + is the only place the phoneme logits exist (preprocess has no logits, and + postprocess only sees the argmax id). It uses only elementwise ops + + ``torch.where`` (no ``.item()`` / host sync), so it stays CUDA-graph safe. + """ # Run in the model dtype (don't force fp32): ``phoneme_final_proj`` weights # follow ``model_config.dtype`` (e.g. bf16), and argmax is dtype-insensitive, # so an fp32 upcast here would mismatch the weight dtype in ``F.linear``. logits = self.phoneme_final_proj(hidden_states[idx]) s = self.arch.phoneme_stacking_factor logits = logits.view(-1, s, self.arch.phoneme_vocab_size) - self._dec_phoneme_tokens[idx] = logits.argmax(dim=-1).long() + preds = logits.argmax(dim=-1).long() # (n, S) + + if self.phoneme_confidence_unk_threshold > 0.0: + max_probs = torch.softmax(logits.float(), dim=-1).amax(dim=-1) # (n, S) + underconfident = (max_probs < self.phoneme_confidence_unk_threshold).any(dim=1, keepdim=True) + eos_step = (preds == self.phoneme_eos_id).any(dim=1, keepdim=True) + replace = underconfident & (~eos_step) + preds = torch.where(replace, torch.full_like(preds, self.phoneme_unk_id), preds) + + self._dec_phoneme_tokens[idx] = preds self._dec_phoneme_valid[idx] = 1 # ------------------------------------------------------------------ @@ -502,6 +543,13 @@ def _preprocess_prefill( device: torch.device, info_dict: dict[str, Any], ) -> tuple[torch.Tensor, torch.Tensor, dict[str, Any]]: + # Forward the audio (local-transformer) sampling params from the request. + # vLLM's ``SamplingParams.temperature`` drives only the dummy backbone + # token sampler, so the real audio temperature/top-k are passed via + # ``additional_information`` and applied to the code predictor here (once, + # at prefill — they are scalars that persist across decode steps). + self._maybe_set_lt_sampling_params(info_dict) + prefill_embeds = self._build_prefill_embeds(device, info_dict) offset = int(info_dict.get("prefill_offset", 0) or 0) @@ -577,6 +625,20 @@ def _build_prefill_embeds( return torch.cat(parts, dim=0) + def _maybe_set_lt_sampling_params(self, info_dict: dict[str, Any]) -> None: + """Apply per-request audio sampling params to the local transformer. + + Reads ``temperature`` / ``top_k`` (alias ``topk``) from the request's + ``additional_information`` and stores them on the code predictor. Absent + keys leave the existing defaults untouched. + """ + temperature = info_dict.get("temperature") + if temperature is not None: + self.code_predictor.temperature = float(self._first_str(temperature) or 0.0) + top_k = info_dict.get("top_k", info_dict.get("topk")) + if top_k is not None: + self.code_predictor.top_k = int(float(self._first_str(top_k) or 0)) + def _get_text_tokenizer(self): """Lazily load the context-text tokenizer from the model directory. @@ -640,11 +702,13 @@ def _preprocess_decode( info_dict: dict[str, Any], ) -> tuple[torch.Tensor, torch.Tensor, dict[str, Any]]: decode_offset = int(info_dict.get("decode_offset", 0) or 0) + info_update: dict[str, Any] = {"decode_offset": decode_offset + 1} - # Text channel (streaming list, one subword consumed per step). Step k + # ── Text channel ── (delay 0: one subword per step from step 0). Step k # consumes text_tokens[k] (the list ends with the text eos id). Once the # stream is exhausted the channel is masked off (adds nothing) rather than - # repeating the last token. + # repeating the last token. The text stream leads the phoneme/audio + # streams by their respective delays. text_tokens = info_dict.get("text_tokens") if isinstance(text_tokens, list) and decode_offset < len(text_tokens): self._dec_text_tokens[start] = int(text_tokens[decode_offset]) @@ -652,29 +716,52 @@ def _preprocess_decode( else: self._dec_text_mask[start] = 0 - # Phoneme channel: previous-step prediction stashed by postprocess. + # ── Phoneme channel ── opens at decode step == ``phonemes_delay`` (seeded + # with phoneme BOS), then feeds back the previous step's prediction, and + # closes one step after the model emits the phoneme EOS (sticky flag). if self.has_phoneme: - last_phon = info_dict.get("last_phoneme_token") - if isinstance(last_phon, torch.Tensor) and last_phon.numel() > 0: - p = last_phon.to(device=device, dtype=torch.long).reshape(-1) - self._dec_phoneme_tokens[start, : p.shape[0]].copy_(p[: self.arch.phoneme_stacking_factor]) + phoneme_ended = bool(info_dict.get("phoneme_ended", False)) + feed_eos = False + if phoneme_ended or decode_offset < self.phonemes_delay: + self._dec_phoneme_valid[start] = 0 + elif decode_offset == self.phonemes_delay: + self._dec_phoneme_tokens[start].fill_(self.phoneme_bos_id) self._dec_phoneme_valid[start] = 1 else: - self._dec_phoneme_valid[start] = 0 - - # Audio channel: previous-frame codes (BOS seed on the first step). - last_codes = info_dict.get("last_audio_codes") - if isinstance(last_codes, torch.Tensor) and last_codes.numel() > 0: - c = last_codes.to(device=device, dtype=torch.long).reshape(-1)[: self.num_codebooks] - self._dec_audio_codes[start, : c.shape[0]].copy_(c) - self._dec_audio_valid[start] = 1 - else: - # First decode step after prefill: seed with audio BOS. + last_phon = info_dict.get("last_phoneme_token") + if isinstance(last_phon, torch.Tensor) and last_phon.numel() > 0: + p = last_phon.to(device=device, dtype=torch.long).reshape(-1)[: self.arch.phoneme_stacking_factor] + self._dec_phoneme_tokens[start, : p.shape[0]].copy_(p) + self._dec_phoneme_valid[start] = 1 + feed_eos = bool((p == self.phoneme_eos_id).any()) + else: + self._dec_phoneme_valid[start] = 0 + if phoneme_ended or feed_eos: + info_update["phoneme_ended"] = True + + # ── Audio channel ── opens at decode step == ``speech_delay`` (seeded with + # audio BOS), then feeds back the previous frame's codes. For the leading + # ``speech_delay`` steps the channel is masked off (only text/phoneme + # condition the backbone); the local transformer still runs for CUDA-graph + # stability but its codes for those frames are discarded by the caller and + # never fed back here. + if decode_offset < self.speech_delay: + self._dec_audio_valid[start] = 0 + elif decode_offset == self.speech_delay: self._dec_audio_codes[start].fill_(self.arch.audio_bos_id) self._dec_audio_valid[start] = 1 + else: + last_codes = info_dict.get("last_audio_codes") + if isinstance(last_codes, torch.Tensor) and last_codes.numel() > 0: + c = last_codes.to(device=device, dtype=torch.long).reshape(-1)[: self.num_codebooks] + self._dec_audio_codes[start, : c.shape[0]].copy_(c) + self._dec_audio_valid[start] = 1 + else: + # Fallback (should not happen once audio has started): seed BOS. + self._dec_audio_codes[start].fill_(self.arch.audio_bos_id) + self._dec_audio_valid[start] = 1 inputs_embeds_out = torch.zeros((1, self.embedding_dim), device=device, dtype=self._combined_embeddings.dtype) - info_update = {"decode_offset": decode_offset + 1} return input_ids, inputs_embeds_out, info_update def postprocess(self, hidden_states: torch.Tensor, multimodal_outputs: Optional[dict[str, Any]] = None, **_: Any): From 9ab003808797c73aa6913bb9327fefbf35331704 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimkov Date: Wed, 3 Jun 2026 12:27:31 +0200 Subject: [PATCH 09/45] examples/tts/easymagpie_vllm_omni: take text as input instead of tokens Signed-off-by: Viacheslav Klimkov --- .../easymagpie_inference_demo.ipynb | 48 +++++++++---------- .../easymagpie_vllm_omni/easymagpie.py | 46 ++++++++++++++++-- 2 files changed, 65 insertions(+), 29 deletions(-) diff --git a/examples/tts/easymagpie_vllm_omni/easymagpie_inference_demo.ipynb b/examples/tts/easymagpie_vllm_omni/easymagpie_inference_demo.ipynb index ebab7bc05532..192e08e1ab03 100644 --- a/examples/tts/easymagpie_vllm_omni/easymagpie_inference_demo.ipynb +++ b/examples/tts/easymagpie_vllm_omni/easymagpie_inference_demo.ipynb @@ -23,9 +23,10 @@ " (`[task_embedding? | speaker_embedding | context_text_embedded]`) and tokenizes\n", " `context_text` itself. `prompt_token_ids = [0] * prompt_len`, sized with\n", " `EasyMagpieTTSForConditionalGeneration.estimate_prompt_len(...)`.\n", - "* **decode** — each step consumes one subword id from the streaming\n", - " `additional_information.text_tokens` list (the tokenized target sentence); the\n", - " local transformer samples all `C * S` stacked audio codebooks for the frame.\n", + "* **decode** — the caller passes the plain target sentence as\n", + " `additional_information.text`; the model tokenizes it in-engine (no caller-side\n", + " tokenization) and consumes one subword id per step. The local transformer\n", + " samples all `C * S` stacked audio codebooks for the frame.\n", "* **output** — per-step audio codes are surfaced on\n", " `OmniOutput.multimodal_outputs[\\\"audio_codes\\\"]` (`BT x num_codebooks`), and the\n", " engine accumulates them across steps just like eartts, so we trim to the last\n", @@ -150,10 +151,10 @@ "downstream consumes it, and `multimodal_output[\\\"audio_codes\\\"]` comes back\n", "`None`).\n", "\n", - "`skip_tokenizer_init: true` — we feed `prompt_token_ids` + `text_tokens`\n", - "directly, so vLLM doesn't need its own tokenizer for the prompt (the model still\n", - "loads the bundled `AutoTokenizer` from `MODEL_DIR` in-engine to tokenize\n", - "`context_text`).\n", + "`skip_tokenizer_init: true` — we feed `prompt_token_ids` directly, so vLLM\n", + "doesn't need its own tokenizer for the prompt. The model loads the bundled\n", + "`AutoTokenizer` from `MODEL_DIR` in-engine and uses it to tokenize both\n", + "`context_text` and the target `text`.\n", "\n", "`max_model_len` must cover `T_ctx` (prefill) + the number of decode steps." ] @@ -191,8 +192,8 @@ " \"max_num_seqs\": 1,\n", " \"model_arch\": \"EasyMagpieTTSForConditionalGeneration\",\n", " \"worker_type\": \"ar\",\n", - " \"scheduler_cls\": \"vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler\",\n", - " \"enforce_eager\": True, # dummy run: skip CUDA-graph capture for a faster start\n", + " \"scheduler_cls\": \"vllm_omni.core.sched.omni_ar_scheduler.OmniARAsyncScheduler\",\n", + " #\"enforce_eager\": True, # dummy run: skip CUDA-graph capture for a faster start\n", " \"trust_remote_code\": True,\n", " \"async_scheduling\": True,\n", " \"enable_prefix_caching\": False,\n", @@ -207,8 +208,8 @@ " \"dtype\": \"float16\",\n", " \"mamba_ssm_cache_dtype\": \"float32\",\n", " \"attention_backend\": \"TRITON_ATTN\",\n", - " # We feed prompt_token_ids + text_tokens directly; the model still\n", - " # loads the bundled AutoTokenizer from MODEL_DIR for context_text.\n", + " # We feed prompt_token_ids directly; the model loads the bundled\n", + " # AutoTokenizer from MODEL_DIR to tokenize context_text + text.\n", " \"skip_tokenizer_init\": True,\n", " },\n", " \"default_sampling_params\": {\n", @@ -254,10 +255,11 @@ " `[task_embedding? | speaker_embedding | context_text_embedded]`.\n", "* **`context_text`** — a plain conditioning string, here `\"[EN]\"`. The model\n", " tokenizes it in-engine and embeds it through the baked `text_embedding` table.\n", - "* **`text_tokens`** `list[int]` — the streaming subword stream: the target\n", - " sentence tokenized with the bundled tokenizer, ending with the model's text\n", - " EOS id. Decode step `k` consumes `text_tokens[k]`; once exhausted the channel\n", - " is masked off (matching the reference `... encode(transcript) + [eos_id]`).\n", + "* **`text`** — the plain target sentence to synthesize. The model tokenizes it\n", + " in-engine at prefill (HF special tokens disabled, trailing text-EOS id\n", + " appended — matching the reference `encode(transcript) + [eos_id]`) and streams\n", + " one subword id per decode step; once exhausted the channel is masked off. No\n", + " caller-side tokenization needed.\n", "\n", "`prompt_token_ids = [0] * prompt_len` are placeholders (the model feeds the\n", "backbone via `inputs_embeds`, never these ids). `prompt_len` must equal the\n", @@ -294,9 +296,9 @@ "# Target sentence to synthesize.\n", "TEXT = \"Hello, this is a test of the EasyMagpie text to speech model.\"\n", "\n", - "# Same tokenizer the engine loads from MODEL_DIR. Used to (a) size the prefill\n", - "# placeholders so prompt_token_ids length matches the assembled context, and\n", - "# (b) tokenize the target sentence into the streaming text stream.\n", + "# Same tokenizer the engine loads from MODEL_DIR. Used only to size the prefill\n", + "# placeholders so prompt_token_ids length matches the assembled context (the\n", + "# target text is tokenized in-engine — we just pass the plain string below).\n", "tokenizer = AutoTokenizer.from_pretrained(MODEL_DIR, trust_remote_code=True)\n", "prompt_len = EasyMagpieTTSForConditionalGeneration.estimate_prompt_len(\n", " speaker_embedding,\n", @@ -305,11 +307,6 @@ " has_task_embedding=arch.num_task_embeddings > 0,\n", ")\n", "\n", - "# Streaming subword ids consumed one per decode step. Mirrors the reference\n", - "# `encode(transcript) + [eos_id]` (no BOS; HF special tokens disabled so the ids\n", - "# index the baked text_embedding table directly).\n", - "text_tokens = tokenizer.encode(TEXT, add_special_tokens=False) + [TEXT_EOS_ID]\n", - "\n", "# Audio (local-transformer) sampling params. vLLM's SamplingParams.temperature\n", "# drives only the dummy backbone token sampler, so the *audio* temperature/top-k\n", "# are forwarded via additional_information. temperature=0.0 == argmax\n", @@ -320,7 +317,7 @@ "additional_information = {\n", " \"speaker_embedding\": speaker_embedding, # (T_audio, embedding_dim) tensor\n", " \"context_text\": CONTEXT_TEXT, # plain string, tokenized in-model\n", - " \"text_tokens\": text_tokens, # list[int], grows by one per step\n", + " \"text\": TEXT, # plain target sentence, tokenized in-model\n", " \"temperature\": LT_TEMPERATURE, # audio sampling temperature (local transformer)\n", " \"top_k\": LT_TOPK, # audio sampling top-k (local transformer)\n", "}\n", @@ -337,8 +334,7 @@ "\n", "print(f\"speaker_embedding : {tuple(speaker_embedding.shape)}\")\n", "print(f\"context_text : {CONTEXT_TEXT!r} -> {tokenizer.encode(CONTEXT_TEXT)}\")\n", - "print(f\"text : {TEXT!r}\")\n", - "print(f\"text_tokens (len {len(text_tokens):3d}) : {text_tokens[:8]}{' ...' if len(text_tokens) > 8 else ''}\")\n", + "print(f\"text : {TEXT!r} (tokenized in-engine)\")\n", "print(f\"prompt_len (placeholders) : {prompt_len}\")\n", "print(f\"decode steps (max_tokens) : {DECODE_STEPS}\")\n", "\n", diff --git a/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/easymagpie.py b/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/easymagpie.py index 8f01eb886bb8..d21f357c36aa 100644 --- a/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/easymagpie.py +++ b/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/easymagpie.py @@ -59,9 +59,18 @@ The caller passes ``prompt_token_ids = [0] * T_ctx``, where ``T_ctx`` is the assembled context length (``[task?] + T_audio + len(tokenize(context_text))``). -* ``text_tokens`` — Python ``list[int]`` of subword ids that grows by one per - decode step; step ``k`` consumes ``text_tokens[k]`` (embedded through the - precomputed per-subword table). +* ``text`` (prefill only) — the plain target sentence to synthesize. This is the + caller's text input: the model tokenizes it in-model at prefill with the + checkpoint's text tokenizer (HF special tokens disabled, trailing text-EOS id + appended), so callers never tokenize themselves. The resulting subword ids are + consumed one per decode step (step ``k`` consumes id ``k``, embedded through + the precomputed per-subword ``text_embedding`` table); once exhausted the text + channel is masked off. + + (Internal: the tokenized ids are stashed as ``text_tokens`` in the per-request + info dict between prefill and decode. A future streaming mode will let the + caller push subword ids gradually instead of one upfront ``text`` string; for + now assume ``text`` is always provided whole at prefill.) * ``temperature`` / ``top_k`` (prefill only, optional) — audio sampling params for the local transformer. vLLM's ``SamplingParams.temperature`` drives only the dummy backbone token sampler, so the *audio* temperature/top-k are passed @@ -187,6 +196,12 @@ def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: text_vocab_size = int(getattr(hf_config, "text_vocab_size", getattr(hf_config, "vocab_size", 0))) self.text_embedding = nn.Embedding(text_vocab_size, self.embedding_dim) + # Text-stream EOS id — the last-but-one row of the text vocab, matching + # the reference ``EasyMagpieTTSInferenceModel.eos_id = num_tokens - 2``. + # Appended to the in-model-tokenized target text stream (see + # :meth:`_encode_text_stream`). + self.text_eos_id = text_vocab_size - 2 + # Task ("service token") embedding — a single learned per-mode row # prepended to the prefill context for multi-mode checkpoints. Built only # when the checkpoint carries one; otherwise ``None``. @@ -570,6 +585,17 @@ def _preprocess_prefill( "prefill_offset": offset + span_len, "decode_offset": 0, } + # Tokenize the caller's ``text`` in-model and stash the subword ids in the + # per-request info dict (alongside the offsets) so each decode step + # consumes one id from it without the caller ever running the tokenizer + # (see :meth:`_preprocess_decode`). The caller always passes ``text`` + # whole at prefill; a future streaming mode will instead let the caller + # push ``text_tokens`` ids gradually, which is why an already-present + # ``text_tokens`` list is left untouched here. + if not info_dict.get("text_tokens"): + text = self._first_str(info_dict.get("text")) + if text: + info_update["text_tokens"] = self._encode_text_stream(text) input_ids_out = torch.full_like(input_ids, _DUMMY_TOKEN_ID) return input_ids_out, take, info_update @@ -663,6 +689,20 @@ def _encode_context_text(self, context_text: str, device: torch.device) -> torch ids = tok.encode(context_text) return torch.tensor(ids, device=device, dtype=torch.long) + def _encode_text_stream(self, text: str) -> list[int]: + """Tokenize the target ``text`` into the streaming subword-id list. + + Mirrors the reference ``tokenizer.encode(transcript) + [eos_id]``: HF + special tokens are disabled so the raw ids index the baked + ``text_embedding`` table directly, and the trailing text-EOS id closes + the stream. One id is consumed per decode step (see + :meth:`_preprocess_decode`); once exhausted the text channel is masked + off. + """ + tok = self._get_text_tokenizer() + ids = tok.encode(text, add_special_tokens=False) + return list(ids) + [self.text_eos_id] + @staticmethod def estimate_prompt_len( speaker_embedding: torch.Tensor, From 36ce9a5fca0be2366effc8fc68dde424f5ba7fbf Mon Sep 17 00:00:00 2001 From: Viacheslav Klimkov Date: Wed, 3 Jun 2026 12:28:16 +0200 Subject: [PATCH 10/45] examples/tts/easymagpie_vllm_omni: add script to benchmark the acoustic token prediction Signed-off-by: Viacheslav Klimkov --- .../benchmark_easymagpie_tts.py | 1082 +++++++++++++++++ 1 file changed, 1082 insertions(+) create mode 100644 examples/tts/easymagpie_vllm_omni/benchmark_easymagpie_tts.py diff --git a/examples/tts/easymagpie_vllm_omni/benchmark_easymagpie_tts.py b/examples/tts/easymagpie_vllm_omni/benchmark_easymagpie_tts.py new file mode 100644 index 000000000000..c8ac80f4e563 --- /dev/null +++ b/examples/tts/easymagpie_vllm_omni/benchmark_easymagpie_tts.py @@ -0,0 +1,1082 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Benchmark the EasyMagpieTTS talker via a single-stage AsyncOmni engine. + +Runs the EasyMagpie talker (``EasyMagpieTTSForConditionalGeneration``) only — +no codec / code2wav — producing stacked audio codes as output. It mirrors the +reference ``qwen3-tts`` talker benchmark and the +``easymagpie_inference_demo.ipynb`` engine setup. + +Metrics measured under configurable concurrency: + +* **TTFT** — time to first decoded frame (first engine token). +* **ITL** — per-token inter-token latency (excluding the first token). +* **E2E** — end-to-end latency per request (up to the audio-EOS frame). +* **RTX** — real-time factor (generated audio seconds / wall time). Both the + per-request RTX and an overall (concurrency-aware) RTX are reported. +* **Throughput** — frames/s and requests/s. + +The decode loop stops at the audio-EOS frame (the EasyMagpie model signals +end-of-speech inside codebook 0 of the codes, not via the vLLM token stream), +so E2E / RTX reflect the real synthesized length rather than the full token +budget. Audio duration is derived from the number of decoded frames: +``audio_seconds = (frames - speech_delay) * frame_stacking_factor / codec_fps``. + +Reads texts from a file (one utterance per line, optionally tab-separated with +the text in the second column) or uses a small built-in default set. + +Usage: + # Basic benchmark with default prompts + python benchmark_easymagpie_tts.py \\ + --model ./easymp_vllm_model \\ + --num-requests 50 + + # From a text file with a concurrency sweep + python benchmark_easymagpie_tts.py \\ + --model ./easymp_vllm_model \\ + --text-file texts.txt \\ + --num-requests 100 \\ + --concurrency 1 4 8 + + # With torch profiler on the run + python benchmark_easymagpie_tts.py \\ + --model ./easymp_vllm_model \\ + --num-requests 20 --concurrency 1 --profile + + # Save JSON results + python benchmark_easymagpie_tts.py \\ + --model ./easymp_vllm_model \\ + --text-file texts.txt \\ + --num-requests 100 --concurrency 1 4 \\ + --result-dir results/ +""" + +import os + +# Keep spawn semantics consistent with the qwen3-tts / eartts demos in case the +# executor backend is switched to a multiproc one. +os.environ.setdefault("VLLM_WORKER_MULTIPROC_METHOD", "spawn") + +import argparse +import asyncio +import json +import logging +import tempfile +import time +import uuid +from dataclasses import asdict, dataclass, field +from datetime import datetime +from pathlib import Path +from typing import Any, Optional + +import numpy as np +import yaml + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(name)s: %(message)s", +) +logger = logging.getLogger(__name__) + +DEFAULT_PROMPTS = [ + "Hello, welcome to the voice synthesis benchmark test.", + "She said she would be here by noon, but nobody showed up.", + "The quick brown fox jumps over the lazy dog near the riverbank.", + "I can't believe how beautiful the sunset looks from up here on the mountain.", + "Please remember to bring your identification documents to the appointment tomorrow morning.", + "Have you ever wondered what it would be like to travel through time and visit ancient civilizations?", + "The restaurant on the corner serves the best pasta I have ever tasted in my entire life.", + "After the meeting, we should discuss the quarterly results and plan for the next phase.", + "Learning a new language takes patience, practice, and a genuine curiosity about other cultures.", + "The train leaves at half past seven, so we need to arrive at the station before then.", + "Could you please turn down the music a little bit, I'm trying to concentrate on my work.", + "It was a dark and stormy night when the old lighthouse keeper heard a knock at the door.", +] + + +# --------------------------------------------------------------------------- +# Stage config generation +# --------------------------------------------------------------------------- + + +def _build_easymagpie_stage_config( + max_num_seqs: int = 1, + profile: bool = False, + torch_profiler_dir: str = "./profiler_traces", + with_stack: bool = False, + record_shapes: bool = False, + gpu_memory_utilization: float = 0.8, + max_model_len: int = 1024, + max_num_batched_tokens: int = 1024, + enforce_eager: bool = False, + max_new_tokens: int = 256, + dtype: str = "float16", + distributed_executor_backend: str = "uni", + cudagraph_mode: Optional[str] = None, +) -> dict: + """Build a single-stage YAML dict containing only the EasyMagpie talker. + + Mirrors the engine_args used in ``easymagpie_inference_demo.ipynb``. + + ``cudagraph_mode`` (when set and ``enforce_eager`` is False) selects the + vLLM CUDA-graph capture strategy via ``compilation_config.cudagraph_mode``: + + * ``FULL_AND_PIECEWISE`` (vLLM default) — a single full graph over the whole + forward for uniform/decode-only batches, piecewise (per compile group: + backbone vs local transformer) for mixed/prefill batches. + * ``PIECEWISE`` — always piecewise, so the backbone and local transformer are + captured as *separate* graphs even during decode. This re-introduces a + launch boundary between them (so decode is a touch slower than FULL), but + makes the backbone-vs-LT split visible as two distinct ``cudaGraphLaunch`` + events in a profiler. + * ``FULL`` / ``FULL_DECODE_ONLY`` — full graph (decode only) capture. + * ``NONE`` — no CUDA graphs (equivalent to ``--enforce-eager``). + """ + engine_args: dict[str, Any] = { + "model_stage": "easymagpie", + "max_num_seqs": max_num_seqs, + "model_arch": "EasyMagpieTTSForConditionalGeneration", + "worker_type": "ar", + "scheduler_cls": "vllm_omni.core.sched.omni_ar_scheduler.OmniARAsyncScheduler", + "enforce_eager": enforce_eager, + "trust_remote_code": True, + "async_scheduling": True, + "enable_prefix_caching": False, + "engine_output_type": "audio", + "gpu_memory_utilization": gpu_memory_utilization, + # "uni" runs the worker in-process (no shm_broadcast IPC); use "mp" + # only when TP/PP > 1 or you actually need a separate worker process. + "distributed_executor_backend": distributed_executor_backend, + "max_num_batched_tokens": max_num_batched_tokens, + "max_model_len": max_model_len, + # bf16/fp16 (not fp32): the Nemotron-H fused-MoE Triton kernel's block + # sizes are tuned for 16-bit and overflow shared memory in fp32. + "dtype": dtype, + "mamba_ssm_cache_dtype": "float32", + "attention_backend": "TRITON_ATTN", + # We feed prompt_token_ids directly; the model loads the bundled + # AutoTokenizer from the model dir to tokenize context_text + text. + "skip_tokenizer_init": True, + } + + # CUDA-graph capture strategy. ``enforce_eager`` already disables graphs, so + # only set compilation_config when graphs are enabled (mirrors the sidecar + # server). Passed as a plain dict so it survives YAML serialization; vLLM + # parses it into a CompilationConfig. + if cudagraph_mode is not None and not enforce_eager: + engine_args["compilation_config"] = {"cudagraph_mode": cudagraph_mode} + + if profile: + engine_args["profiler_config"] = { + "profiler": "torch", + "torch_profiler_dir": os.path.abspath(torch_profiler_dir), + "torch_profiler_with_stack": with_stack, + "torch_profiler_record_shapes": record_shapes, + } + + cfg = { + "stage_args": [ + { + "stage_id": 0, + "stage_type": "llm", + "is_comprehension": True, + "final_output": True, + # "audio" (not "latent") is required for a single-stage AR TTS + # model: it makes the AR model runner attach the per-step + # multimodal payload ("audio_codes") to the output so the codes + # reach the client. + "final_output_type": "audio", + "runtime": {"devices": "0"}, + "engine_args": engine_args, + "default_sampling_params": { + # The backbone token sampler is a no-op (audio is sampled in + # the local transformer); the audio temperature/top-k are + # forwarded per-request via additional_information. + "temperature": 0.0, + "max_tokens": max_new_tokens, + "detokenize": False, + # Audio EOS lives in the codes, not the vLLM token stream, so + # let the budget run and stop client-side at the EOS frame. + "ignore_eos": True, + }, + } + ], + } + return cfg + + +def _write_temp_stage_config(cfg: dict) -> str: + """Write stage config dict to a temp YAML file, return its path.""" + tmp = tempfile.NamedTemporaryFile( + mode="w", + suffix=".yaml", + prefix="easymagpie_bench_", + delete=False, + ) + yaml.dump(cfg, tmp, default_flow_style=False, sort_keys=False) + tmp.close() + logger.info("Wrote single-stage config to %s", tmp.name) + return tmp.name + + +# --------------------------------------------------------------------------- +# Model metadata (arch scalars + tokenizer + speaker embedding) +# --------------------------------------------------------------------------- + + +@dataclass +class ModelMeta: + """Scalars + assets needed to build prompts and interpret outputs.""" + + arch: Any + tokenizer: Any + speaker_embedding: Any # torch.Tensor (T_audio, embedding_dim) + prompt_len: int + audio_eos_id: int + speech_delay: int + frame_stacking_factor: int + + +def _load_model_meta( + model_dir: str, + speaker: str, + speaker_embedding_path: Optional[str], + context_text: str, +) -> ModelMeta: + """Read config.json, tokenizer, and the speaker embedding from the model dir. + + Mirrors the prompt-prep cells of ``easymagpie_inference_demo.ipynb``: the + arch scalars come from ``config.json``, the speaker embedding from + ``speaker_embeddings/.pt``, and the prefill placeholder length from + ``EasyMagpieTTSForConditionalGeneration.estimate_prompt_len(...)``. + """ + import torch + from transformers import AutoTokenizer + + from easymagpie_vllm_omni.config import EasyMagpieOmniArch + from easymagpie_vllm_omni.easymagpie import EasyMagpieTTSForConditionalGeneration + + model_path = Path(model_dir) + config = json.loads((model_path / "config.json").read_text()) + arch = EasyMagpieOmniArch.from_hf_config(type("Cfg", (), config)) + + # Speaker-encoded context audio (audio branch of prepare_context_tensors). + if speaker_embedding_path is not None: + emb_path = Path(speaker_embedding_path) + else: + emb_path = model_path / "speaker_embeddings" / f"{speaker}.pt" + if not emb_path.exists(): + raise FileNotFoundError(f"Speaker embedding not found: {emb_path}") + loaded = torch.load(emb_path, map_location="cpu") + speaker_embedding = loaded["speaker_encoding"] if isinstance(loaded, dict) else loaded + speaker_embedding = speaker_embedding.to(torch.float32) + + tokenizer = AutoTokenizer.from_pretrained(model_dir, trust_remote_code=True) + + # prompt_len depends only on the speaker embedding + context_text (+ task + # embedding) — NOT on the target text (which is streamed in-engine), so we + # size it once. + prompt_len = EasyMagpieTTSForConditionalGeneration.estimate_prompt_len( + speaker_embedding, + tokenize=lambda t: tokenizer.encode(t), + context_text=context_text, + has_task_embedding=arch.num_task_embeddings > 0, + ) + + return ModelMeta( + arch=arch, + tokenizer=tokenizer, + speaker_embedding=speaker_embedding, + prompt_len=int(prompt_len), + audio_eos_id=int(arch.audio_eos_id), + speech_delay=int(getattr(arch, "streaming_speech_delay", 0) or 0), + frame_stacking_factor=int(arch.frame_stacking_factor), + ) + + +def build_prompt( + text: str, + meta: ModelMeta, + context_text: str, + lt_temperature: float, + lt_topk: int, +) -> dict: + """Build an engine input dict from a target sentence + the shared assets.""" + additional_information = { + "speaker_embedding": meta.speaker_embedding, # (T_audio, embedding_dim) + "context_text": context_text, # plain string, tokenized in-model + "text": text, # plain target sentence, tokenized in-model + "temperature": lt_temperature, # audio sampling temperature (local transformer) + "top_k": lt_topk, # audio sampling top-k (local transformer) + } + return { + "prompt_token_ids": [0] * meta.prompt_len, + "additional_information": additional_information, + } + + +# --------------------------------------------------------------------------- +# Result dataclasses +# --------------------------------------------------------------------------- + + +@dataclass +class RequestResult: + success: bool = False + text: str = "" + prompt_len: int = 0 + num_generated: int = 0 # decoded frames (engine tokens) up to EOS + audio_frames: int = 0 # codec frames of real audio (post speech-delay, pre-EOS) + audio_s: float = 0.0 # synthesized audio duration in seconds + steps: int = 0 + eos_reached: bool = False + ttft_s: float = 0.0 + e2e_s: float = 0.0 + rtx: float = 0.0 # audio_s / e2e_s + inter_token_latencies: list = field(default_factory=list) + error: str = "" + + +@dataclass +class BenchmarkResult: + config_name: str = "" + concurrency: int = 0 + num_requests: int = 0 + completed: int = 0 + failed: int = 0 + duration_s: float = 0.0 + # TTFT + mean_ttft_ms: float = 0.0 + median_ttft_ms: float = 0.0 + p95_ttft_ms: float = 0.0 + p99_ttft_ms: float = 0.0 + # E2E + mean_e2e_ms: float = 0.0 + median_e2e_ms: float = 0.0 + p95_e2e_ms: float = 0.0 + p99_e2e_ms: float = 0.0 + # ITL (inter-token latency, excluding first token) + mean_itl_ms: float = 0.0 + median_itl_ms: float = 0.0 + p95_itl_ms: float = 0.0 + p99_itl_ms: float = 0.0 + # RTX (real-time factor: synthesized audio seconds / generation seconds) + mean_rtx: float = 0.0 + median_rtx: float = 0.0 + overall_rtx: float = 0.0 # total_audio_s / wall_clock_duration (concurrency-aware) + # Throughput + total_tokens: int = 0 + total_audio_s: float = 0.0 + mean_tokens_per_request: float = 0.0 + token_throughput: float = 0.0 + request_throughput: float = 0.0 + per_request: list = field(default_factory=list) + + +# --------------------------------------------------------------------------- +# Inference +# --------------------------------------------------------------------------- + + +def _extract_request_output(stage_output): + """Return the RequestOutput-like object from a yielded stage output. + + AsyncOmni stages may yield either a wrapper carrying ``.request_output`` + (qwen3-tts style) or the RequestOutput directly (easymagpie demo style). + """ + return getattr(stage_output, "request_output", stage_output) + + +async def run_one_request( + omni, + prompt: dict, + sampling_params, + request_id: str, + meta: ModelMeta, + codec_fps: float, + stop_on_eos: bool, +) -> RequestResult: + """Submit one TTS request, collect per-token timing and audio length. + + Each engine step yields one decoded frame (one layer-0 token). We time the + first token (TTFT) and the gaps between subsequent tokens (ITL). The audio + EOS lives in codebook 0 of the accumulated ``audio_codes`` (not in the vLLM + token stream), so we watch the newest decoded frame and stop at the EOS + frame to recover the real synthesized length. + """ + import torch + + result = RequestResult() + t_start = time.perf_counter() + t_last_token = None + prev_num_tokens = 0 + eos_decode_idx = None # 0-based decode-frame index where audio EOS appears + + try: + gen = omni.generate( + prompt, + sampling_params_list=[sampling_params], + request_id=request_id, + ) + async for stage_output in gen: + now = time.perf_counter() + ro = _extract_request_output(stage_output) + result.steps += 1 + + cur_num_tokens = prev_num_tokens + if hasattr(ro, "outputs") and ro.outputs: + out0 = ro.outputs[0] + cum_ids = getattr(out0, "cumulative_token_ids", None) + if cum_ids is not None: + cur_num_tokens = len(cum_ids) + else: + cur_num_tokens = len(getattr(out0, "token_ids", []) or []) + + if cur_num_tokens > prev_num_tokens: + if t_last_token is None: + result.ttft_s = now - t_start + else: + result.inter_token_latencies.append(now - t_last_token) + t_last_token = now + + # Audio-EOS detection on the newest decoded frame. The accumulated + # audio_codes hold (T_ctx prefill + decode) rows; the last row is + # the newest decoded frame. Only meaningful past the speech delay. + mm = getattr(stage_output, "multimodal_output", None) or {} + audio_codes = mm.get("audio_codes") + newest_frame_idx = cur_num_tokens - 1 # 0-based decode-frame index + if ( + eos_decode_idx is None + and newest_frame_idx >= meta.speech_delay + and isinstance(audio_codes, torch.Tensor) + and audio_codes.numel() > 0 + ): + if int(audio_codes[-1, 0]) == meta.audio_eos_id: + eos_decode_idx = newest_frame_idx + result.eos_reached = True + + prev_num_tokens = cur_num_tokens + + if eos_decode_idx is not None and stop_on_eos: + break + + t_end = time.perf_counter() + result.e2e_s = t_end - t_start + result.num_generated = prev_num_tokens + result.success = True + + if result.ttft_s == 0.0 and result.steps > 0: + result.ttft_s = t_end - t_start + + # Real audio length: frames between the start of speech (speech_delay) + # and the EOS frame (or the full decode if no EOS was emitted). + last_audio_frame = eos_decode_idx if eos_decode_idx is not None else prev_num_tokens + result.audio_frames = max(0, last_audio_frame - meta.speech_delay) + if codec_fps > 0: + result.audio_s = result.audio_frames * meta.frame_stacking_factor / codec_fps + result.rtx = result.audio_s / result.e2e_s if result.e2e_s > 0 else 0.0 + + except Exception as exc: + result.e2e_s = time.perf_counter() - t_start + result.error = str(exc) + logger.error("Request %s failed: %s", request_id, exc) + finally: + # Make sure the async generator is closed (aborts the request in the + # engine when we broke out early on EOS). + try: + await gen.aclose() + except Exception: + pass + + return result + + +# --------------------------------------------------------------------------- +# Worker / concurrency +# --------------------------------------------------------------------------- + + +async def worker( + worker_id: int, + omni, + texts: list, + meta: ModelMeta, + context_text: str, + lt_temperature: float, + lt_topk: int, + sampling_params, + codec_fps: float, + stop_on_eos: bool, + results: list, + counter: dict, + lock: asyncio.Lock, +): + """Persistent async worker that picks texts until the quota is exhausted.""" + while True: + async with lock: + if counter["remaining"] <= 0: + break + counter["remaining"] -= 1 + idx = counter["issued"] + counter["issued"] += 1 + + text = texts[idx % len(texts)] + request_id = f"bench-easymp-w{worker_id}-{uuid.uuid4().hex[:8]}" + + prompt = build_prompt( + text=text, + meta=meta, + context_text=context_text, + lt_temperature=lt_temperature, + lt_topk=lt_topk, + ) + + result = await run_one_request( + omni, + prompt, + sampling_params, + request_id, + meta, + codec_fps, + stop_on_eos, + ) + result.text = text + result.prompt_len = len(prompt["prompt_token_ids"]) + + async with lock: + results.append(result) + done = len(results) + + if done % 10 == 0 or done == counter["total"]: + logger.info(" progress: %d / %d", done, counter["total"]) + + +# --------------------------------------------------------------------------- +# Metrics +# --------------------------------------------------------------------------- + + +def _pct(arr, p): + return float(np.percentile(arr, p)) if len(arr) > 0 else 0.0 + + +def compute_and_print_metrics( + results: list, + duration: float, + concurrency: int, + num_requests: int, +) -> BenchmarkResult: + successful = [r for r in results if r.success] + failed = [r for r in results if not r.success] + + bench = BenchmarkResult( + concurrency=concurrency, + num_requests=num_requests, + completed=len(successful), + failed=len(failed), + duration_s=duration, + ) + + if not successful: + print("ERROR: No requests completed successfully.") + return bench + + ttfts = [r.ttft_s * 1000 for r in successful] + e2es = [r.e2e_s * 1000 for r in successful] + rtxs = [r.rtx for r in successful] + all_itls = [] + for r in successful: + all_itls.extend([t * 1000 for t in r.inter_token_latencies]) + gen_tokens = [r.num_generated for r in successful] + + bench.mean_ttft_ms = float(np.mean(ttfts)) + bench.median_ttft_ms = float(np.median(ttfts)) + bench.p95_ttft_ms = _pct(ttfts, 95) + bench.p99_ttft_ms = _pct(ttfts, 99) + + bench.mean_e2e_ms = float(np.mean(e2es)) + bench.median_e2e_ms = float(np.median(e2es)) + bench.p95_e2e_ms = _pct(e2es, 95) + bench.p99_e2e_ms = _pct(e2es, 99) + + if all_itls: + bench.mean_itl_ms = float(np.mean(all_itls)) + bench.median_itl_ms = float(np.median(all_itls)) + bench.p95_itl_ms = _pct(all_itls, 95) + bench.p99_itl_ms = _pct(all_itls, 99) + + bench.mean_rtx = float(np.mean(rtxs)) + bench.median_rtx = float(np.median(rtxs)) + + bench.total_tokens = int(sum(gen_tokens)) + bench.total_audio_s = float(sum(r.audio_s for r in successful)) + bench.mean_tokens_per_request = float(np.mean(gen_tokens)) + bench.token_throughput = bench.total_tokens / duration if duration > 0 else 0.0 + bench.request_throughput = len(successful) / duration if duration > 0 else 0.0 + bench.overall_rtx = bench.total_audio_s / duration if duration > 0 else 0.0 + + bench.per_request = [ + { + "ttft_ms": r.ttft_s * 1000, + "e2e_ms": r.e2e_s * 1000, + "rtx": r.rtx, + "num_generated": r.num_generated, + "audio_frames": r.audio_frames, + "audio_s": r.audio_s, + "eos_reached": r.eos_reached, + "steps": r.steps, + "prompt_len": r.prompt_len, + "mean_itl_ms": float(np.mean([t * 1000 for t in r.inter_token_latencies])) + if r.inter_token_latencies + else 0.0, + "text": r.text, + } + for r in successful + ] + + eos_hits = sum(1 for r in successful if r.eos_reached) + + W = 56 + print(f"\n{'=' * W}") + print(f"{'Benchmark Result':^{W}}") + print(f"{'=' * W}") + print(f"{'Successful requests:':<42}{bench.completed}") + print(f"{'Failed requests:':<42}{bench.failed}") + print(f"{'Reached audio EOS:':<42}{eos_hits} / {bench.completed}") + print(f"{'Concurrency:':<42}{concurrency}") + print(f"{'Wall-clock duration (s):':<42}{duration:.2f}") + print(f"{'Request throughput (req/s):':<42}{bench.request_throughput:.2f}") + + print(f"\n{'-' * W}") + print(f"{'Time to First Token (TTFT)':^{W}}") + print(f"{'-' * W}") + print(f"{'Mean (ms):':<42}{bench.mean_ttft_ms:.2f}") + print(f"{'Median (ms):':<42}{bench.median_ttft_ms:.2f}") + print(f"{'P95 (ms):':<42}{bench.p95_ttft_ms:.2f}") + print(f"{'P99 (ms):':<42}{bench.p99_ttft_ms:.2f}") + + print(f"\n{'-' * W}") + print(f"{'End-to-End Latency (E2E)':^{W}}") + print(f"{'-' * W}") + print(f"{'Mean (ms):':<42}{bench.mean_e2e_ms:.2f}") + print(f"{'Median (ms):':<42}{bench.median_e2e_ms:.2f}") + print(f"{'P95 (ms):':<42}{bench.p95_e2e_ms:.2f}") + print(f"{'P99 (ms):':<42}{bench.p99_e2e_ms:.2f}") + + print(f"\n{'-' * W}") + print(f"{'Inter-Token Latency (ITL)':^{W}}") + print(f"{'-' * W}") + if all_itls: + print(f"{'Mean (ms):':<42}{bench.mean_itl_ms:.2f}") + print(f"{'Median (ms):':<42}{bench.median_itl_ms:.2f}") + print(f"{'P95 (ms):':<42}{bench.p95_itl_ms:.2f}") + print(f"{'P99 (ms):':<42}{bench.p99_itl_ms:.2f}") + else: + print(f"{'(no inter-token data)':^{W}}") + + print(f"\n{'-' * W}") + print(f"{'Real-Time Factor (RTX = audio_s / gen_s)':^{W}}") + print(f"{'-' * W}") + print(f"{'Mean RTX (per request):':<42}{bench.mean_rtx:.2f}x") + print(f"{'Median RTX (per request):':<42}{bench.median_rtx:.2f}x") + print(f"{'Overall RTX (total audio / wall):':<42}{bench.overall_rtx:.2f}x") + + print(f"\n{'-' * W}") + print(f"{'Throughput':^{W}}") + print(f"{'-' * W}") + print(f"{'Total frames generated:':<42}{bench.total_tokens}") + print(f"{'Total audio generated (s):':<42}{bench.total_audio_s:.2f}") + print(f"{'Mean frames / request:':<42}{bench.mean_tokens_per_request:.1f}") + print(f"{'Frame throughput (frames/s):':<42}{bench.token_throughput:.2f}") + print(f"{'=' * W}\n") + + if failed: + print(f" First {min(3, len(failed))} errors:") + for r in failed[:3]: + print(f" {r.error[:200]}") + + return bench + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + + +async def main(args): + from vllm import SamplingParams + from vllm_omni import AsyncOmni + + model_name = args.model + + # ── Load texts ──────────────────────────────────────────────────────── + if args.text_file: + path = Path(args.text_file) + if not path.exists(): + print(f"ERROR: text file not found: {path}") + return + raw_lines = [line.strip() for line in path.read_text().splitlines() if line.strip()] + texts = [] + for line in raw_lines: + if "\t" in line: + texts.append(line.split("\t", 1)[1].strip()) + else: + texts.append(line) + texts = [t for t in texts if t] + logger.info("Loaded %d texts from %s", len(texts), path) + else: + texts = DEFAULT_PROMPTS + logger.info("Using %d default prompts", len(texts)) + + if not texts: + print("ERROR: no texts available.") + return + + # ── Read arch scalars + tokenizer + speaker embedding ───────────────── + logger.info("Reading model metadata from %s ...", model_name) + meta = _load_model_meta( + model_dir=model_name, + speaker=args.speaker, + speaker_embedding_path=args.speaker_embedding, + context_text=args.context_text, + ) + logger.info( + "prompt_len=%d audio_eos_id=%d speech_delay=%d frame_stacking=%d", + meta.prompt_len, + meta.audio_eos_id, + meta.speech_delay, + meta.frame_stacking_factor, + ) + if meta.prompt_len + args.max_new_tokens > args.max_model_len: + logger.warning( + "prompt_len (%d) + max_new_tokens (%d) exceeds max_model_len (%d); raise --max-model-len.", + meta.prompt_len, + args.max_new_tokens, + args.max_model_len, + ) + + max_concurrency = max(args.concurrency) + + # ── Build stage config ──────────────────────────────────────────────── + stage_cfg = _build_easymagpie_stage_config( + max_num_seqs=max_concurrency, + profile=args.profile, + torch_profiler_dir=args.torch_profiler_dir, + with_stack=args.with_stack, + record_shapes=args.record_shapes, + gpu_memory_utilization=args.gpu_memory_utilization, + max_model_len=args.max_model_len, + max_num_batched_tokens=args.max_num_batched_tokens, + enforce_eager=args.enforce_eager, + max_new_tokens=args.max_new_tokens, + dtype=args.dtype, + distributed_executor_backend=args.distributed_executor_backend, + cudagraph_mode=args.cudagraph_mode, + ) + if args.cudagraph_mode is not None and args.enforce_eager: + logger.warning( + "--cudagraph-mode %s is ignored because --enforce-eager disables CUDA graphs.", + args.cudagraph_mode, + ) + elif args.cudagraph_mode is not None: + logger.info("CUDA-graph mode: %s", args.cudagraph_mode) + tmp_config_path = _write_temp_stage_config(stage_cfg) + + sampling_params = SamplingParams( + temperature=0.0, + max_tokens=args.max_new_tokens, + detokenize=False, + ignore_eos=True, + ) + + try: + logger.info("Creating AsyncOmni engine (EasyMagpie talker only) for %s ...", model_name) + omni = AsyncOmni( + model=model_name, + stage_configs_path=tmp_config_path, + log_stats=args.log_stats, + stage_init_timeout=args.stage_init_timeout, + ) + logger.info("Engine ready (single stage: EasyMagpie talker).") + + all_bench_results = [] + + for concurrency in args.concurrency: + logger.info( + "=== concurrency=%d requests=%d ===", + concurrency, + args.num_requests, + ) + + # ── Warmup ──────────────────────────────────────────────────── + warmup_count = 0 if args.no_warmup else args.num_warmups * concurrency + if warmup_count > 0: + logger.info("Warming up with %d requests (concurrency=%d)...", warmup_count, concurrency) + warmup_results: list = [] + warmup_counter = { + "remaining": warmup_count, + "issued": 0, + "total": warmup_count, + } + warmup_lock = asyncio.Lock() + warmup_tasks = [ + asyncio.create_task( + worker( + worker_id=i, + omni=omni, + texts=texts, + meta=meta, + context_text=args.context_text, + lt_temperature=args.lt_temperature, + lt_topk=args.lt_topk, + sampling_params=sampling_params, + codec_fps=args.codec_frame_rate, + stop_on_eos=not args.no_stop_on_eos, + results=warmup_results, + counter=warmup_counter, + lock=warmup_lock, + ) + ) + for i in range(concurrency) + ] + await asyncio.gather(*warmup_tasks) + warmup_ok = sum(1 for r in warmup_results if r.success) + logger.info("Warmup done: %d / %d succeeded.", warmup_ok, warmup_count) + + # ── Benchmark run ───────────────────────────────────────────── + logger.info("Starting benchmark run (%d requests, concurrency=%d)...", args.num_requests, concurrency) + + bench_results: list = [] + counter = { + "remaining": args.num_requests, + "issued": 0, + "total": args.num_requests, + } + lock = asyncio.Lock() + + if args.profile: + logger.info("Starting profiler ...") + await omni.start_profile( + profile_prefix=args.profile_prefix, + stages=[0], + ) + + start_time = time.perf_counter() + try: + tasks = [ + asyncio.create_task( + worker( + worker_id=i, + omni=omni, + texts=texts, + meta=meta, + context_text=args.context_text, + lt_temperature=args.lt_temperature, + lt_topk=args.lt_topk, + sampling_params=sampling_params, + codec_fps=args.codec_frame_rate, + stop_on_eos=not args.no_stop_on_eos, + results=bench_results, + counter=counter, + lock=lock, + ) + ) + for i in range(concurrency) + ] + await asyncio.gather(*tasks) + finally: + if args.profile: + logger.info("Stopping profiler ...") + await omni.stop_profile(stages=[0]) + + duration = time.perf_counter() - start_time + + bench = compute_and_print_metrics( + bench_results, + duration, + concurrency, + args.num_requests, + ) + bench.config_name = args.config_name + all_bench_results.append(asdict(bench)) + + # ── Save results ────────────────────────────────────────────────── + if args.result_dir: + result_dir = Path(args.result_dir) + result_dir.mkdir(parents=True, exist_ok=True) + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + result_file = result_dir / f"bench_easymagpie_{args.config_name}_{timestamp}.json" + with open(result_file, "w") as f: + json.dump(all_bench_results, f, indent=2) + logger.info("Results saved to %s", result_file) + + omni.shutdown() + finally: + os.unlink(tmp_config_path) + + logger.info("Done.") + + +def parse_args(): + parser = argparse.ArgumentParser( + description="Benchmark the EasyMagpieTTS talker (AR stage only) via AsyncOmni", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=__doc__, + ) + + model = parser.add_argument_group("model / input") + model.add_argument( + "--model", + type=str, + default="./easymp_vllm_model", + help="Converted EasyMagpie model directory (output of easy_magpietts_convert_to_vllm.py)", + ) + model.add_argument( + "--text-file", + type=str, + default=None, + help="Path to text file (one utterance per line, optionally tab-separated with text in 2nd column)", + ) + model.add_argument( + "--speaker", + type=str, + default="eng", + help="Speaker embedding name under /speaker_embeddings/.pt", + ) + model.add_argument( + "--speaker-embedding", + type=str, + default=None, + help="Explicit path to a speaker embedding .pt (overrides --speaker)", + ) + model.add_argument( + "--context-text", + type=str, + default="[EN]", + help="Conditioning string tokenized + embedded in-engine (e.g. '[EN]')", + ) + model.add_argument( + "--lt-temperature", + type=float, + default=0.0, + help="Audio (local-transformer) sampling temperature (0.0 == argmax)", + ) + model.add_argument( + "--lt-topk", + type=int, + default=80, + help="Audio (local-transformer) sampling top-k", + ) + model.add_argument( + "--max-new-tokens", + type=int, + default=256, + help="Max decode frames per request (decode budget; trimmed at audio EOS)", + ) + model.add_argument( + "--codec-frame-rate", + type=float, + default=25.0, + help="Codec frame rate (Hz) used to convert decoded frames to audio seconds " + "(default 25 for the 25fps spectral codec)", + ) + + bench = parser.add_argument_group("benchmark") + bench.add_argument( + "-c", + "--concurrency", + type=int, + nargs="+", + default=[1], + help="Concurrency levels to test (space-separated, default: 1)", + ) + bench.add_argument( + "-n", + "--num-requests", + type=int, + default=50, + help="Total number of requests per concurrency level (default: 50)", + ) + bench.add_argument( + "--num-warmups", + type=int, + default=3, + help="Warmup rounds per concurrency level (total warmup = concurrency * this, default: 3)", + ) + bench.add_argument("--no-warmup", action="store_true", help="Skip warmup") + bench.add_argument( + "--no-stop-on-eos", + action="store_true", + help="Do not stop at the audio-EOS frame; run the full decode budget every request", + ) + bench.add_argument( + "--config-name", + type=str, + default="easymagpie", + help="Label for this run (used in result filenames)", + ) + bench.add_argument( + "--result-dir", + type=str, + default=None, + help="Directory to save JSON results", + ) + + engine = parser.add_argument_group("engine") + engine.add_argument("--gpu-memory-utilization", type=float, default=0.8) + engine.add_argument("--max-model-len", type=int, default=1024) + engine.add_argument("--max-num-batched-tokens", type=int, default=1024) + engine.add_argument("--dtype", type=str, default="float16", help="Model dtype (float16 / bfloat16)") + engine.add_argument("--enforce-eager", action="store_true") + engine.add_argument( + "--cudagraph-mode", + type=str, + default=None, + choices=["NONE", "PIECEWISE", "FULL", "FULL_DECODE_ONLY", "FULL_AND_PIECEWISE"], + help="vLLM CUDA-graph capture strategy (compilation_config.cudagraph_mode). " + "Default: unset (vLLM default, FULL_AND_PIECEWISE). Use PIECEWISE to capture the " + "backbone and local transformer as separate graphs during decode so their split is " + "visible in a profiler (slightly slower than the default full decode graph). " + "Ignored when --enforce-eager is set.", + ) + engine.add_argument("--stage-init-timeout", type=int, default=300) + engine.add_argument("--log-stats", action="store_true", default=False) + engine.add_argument( + "--distributed-executor-backend", + type=str, + default="uni", + choices=["uni", "mp", "ray"], + help="vLLM executor backend. 'uni' runs the worker in-process and " + "avoids shm_broadcast IPC round-trips (recommended for TP=1, single " + "GPU). Default: uni.", + ) + + prof = parser.add_argument_group("profiling") + prof.add_argument( + "--profile", + action="store_true", + help="Enable torch profiler during the benchmark run", + ) + prof.add_argument("--profile-prefix", type=str, default=None, help="Prefix for profiler trace filenames") + prof.add_argument( + "--torch-profiler-dir", type=str, default="./profiler_traces", help="Directory for torch profiler traces" + ) + prof.add_argument("--with-stack", action="store_true", help="Record Python call stacks in profiler") + prof.add_argument("--record-shapes", action="store_true", help="Record tensor shapes in profiler") + + return parser.parse_args() + + +if __name__ == "__main__": + asyncio.run(main(parse_args())) From f5c06a519c2c942cdc095d21e05df98329f7c582 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimkov Date: Wed, 3 Jun 2026 14:52:04 +0200 Subject: [PATCH 11/45] examples/tts/easymagpie_vllm_omni/easy_magpietts_convert_to_vllm.py: do ckpt conversion without precision loss Signed-off-by: Viacheslav Klimkov --- .../tts/easymagpie_vllm_omni/easy_magpietts_convert_to_vllm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/tts/easymagpie_vllm_omni/easy_magpietts_convert_to_vllm.py b/examples/tts/easymagpie_vllm_omni/easy_magpietts_convert_to_vllm.py index 4cb99a08baee..af5cbe720f04 100644 --- a/examples/tts/easymagpie_vllm_omni/easy_magpietts_convert_to_vllm.py +++ b/examples/tts/easymagpie_vllm_omni/easy_magpietts_convert_to_vllm.py @@ -157,7 +157,7 @@ def parse_args(): parser.add_argument("--context_audio_duration", type=float, default=5.0) parser.add_argument( "--dtype", - default="bfloat16", + default="float32", choices=["bfloat16", "float16", "float32"], help="Saved weight dtype / config torch_dtype. bf16 matches the reference inference setup.", ) From 8721e54e888ee86dce93ccb6f3e47f3e87128647 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimkov Date: Wed, 3 Jun 2026 16:16:59 +0200 Subject: [PATCH 12/45] examples/tts/easymagpie_vllm_omni/tests: add tests to check equivalence of cudagraph-friendly LT re-implemantation Signed-off-by: Viacheslav Klimkov --- .../easymagpie_vllm_omni/tests/conftest.py | 78 ++++++ .../easymagpie_vllm_omni/tests/test_config.py | 88 +++++++ .../tests/test_local_transformer.py | 242 ++++++++++++++++++ 3 files changed, 408 insertions(+) create mode 100644 examples/tts/easymagpie_vllm_omni/tests/conftest.py create mode 100644 examples/tts/easymagpie_vllm_omni/tests/test_config.py create mode 100644 examples/tts/easymagpie_vllm_omni/tests/test_local_transformer.py diff --git a/examples/tts/easymagpie_vllm_omni/tests/conftest.py b/examples/tts/easymagpie_vllm_omni/tests/conftest.py new file mode 100644 index 000000000000..8170bd62b34c --- /dev/null +++ b/examples/tts/easymagpie_vllm_omni/tests/conftest.py @@ -0,0 +1,78 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Shared pytest fixtures for the EasyMagpieTTS vLLM-Omni tests. + +The model definition (``easymagpie_vllm_omni.local_transformer`` etc.) is plain +PyTorch: the ``@support_torch_compile`` decorator short-circuits to eager when +``compilation_config.mode == CompilationMode.NONE``, and the modules only read a +handful of scalars off the ``VllmConfig``. So the whole stack can be exercised as +ordinary PyTorch with a tiny stand-in config — **no model directory, no engine, +no GPU required** — which is what these fixtures provide. + +All heavy imports (torch / vllm) are done lazily inside the fixture so test +collection never fails on machines where those packages are absent; the +dependent tests ``importorskip`` them and are skipped instead. +""" +from __future__ import annotations + +import types + +import pytest + +# A deliberately tiny architecture so the tests run fast on CPU. Dimensions are +# kept equal by default (so the in/out projections collapse to ``nn.Identity``, +# matching the reference SmallMamba checkpoint where everything is 1536-wide). +_DEFAULT_ARCH: dict = dict( + hidden_dim=64, + embedding_dim=64, + audio_embedding_dim=64, + num_audio_codebooks=2, + codebook_size=32, + frame_stacking_factor=2, + local_transformer_n_layers=2, + local_transformer_n_heads=4, + local_transformer_hidden_dim=64, +) + + +def build_vllm_config(**arch_overrides): + """Build a minimal stand-in ``VllmConfig`` for the code predictor. + + Returns a ``types.SimpleNamespace`` exposing exactly the attributes the + EasyMagpie modules touch at construction time: + + * ``model_config.hf_config`` — arch scalars (read via ``from_hf_config``); + * ``model_config.dtype`` — buffer dtype; + * ``scheduler_config.max_num_batched_tokens`` — scratch-buffer length; + * ``compilation_config.mode`` — ``CompilationMode.NONE`` so the + ``@support_torch_compile`` wrapper stays in eager mode. + + Any keyword overrides are merged into the default tiny arch profile. + """ + import torch + from vllm.config import CompilationMode + + arch = {**_DEFAULT_ARCH, **arch_overrides} + hf_config = types.SimpleNamespace(**arch) + return types.SimpleNamespace( + model_config=types.SimpleNamespace(hf_config=hf_config, dtype=torch.float32), + scheduler_config=types.SimpleNamespace(max_num_batched_tokens=128), + compilation_config=types.SimpleNamespace(mode=CompilationMode.NONE), + ) + + +@pytest.fixture +def vllm_config_factory(): + """Fixture returning the :func:`build_vllm_config` factory.""" + return build_vllm_config diff --git a/examples/tts/easymagpie_vllm_omni/tests/test_config.py b/examples/tts/easymagpie_vllm_omni/tests/test_config.py new file mode 100644 index 000000000000..e8955d348328 --- /dev/null +++ b/examples/tts/easymagpie_vllm_omni/tests/test_config.py @@ -0,0 +1,88 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Pure-Python tests for :class:`EasyMagpieOmniArch`. + +These have no heavy dependencies (no torch / vllm) and validate the derived +quantities and the ``from_hf_config`` merge logic that the rest of the model +relies on for correct vocab sizes and special-token ids. +""" +from __future__ import annotations + +import types + +from easymagpie_vllm_omni.config import ( + EASYMAGPIE_SMALLMAMBA, + NUM_SPECIAL_AUDIO_TOKENS, + SPECIAL_AUDIO_EOS, + SPECIAL_AUDIO_MASK, + EasyMagpieOmniArch, +) + + +def test_derived_codebook_counts(): + arch = EasyMagpieOmniArch(num_audio_codebooks=8, frame_stacking_factor=2, codebook_size=1024) + assert arch.num_stacked_codebooks == 16 + assert arch.num_all_tokens_per_codebook == 1024 + NUM_SPECIAL_AUDIO_TOKENS + + +def test_special_token_ids_default_to_codebook_offsets(): + arch = EasyMagpieOmniArch(codebook_size=1024) + assert arch.audio_eos_id == 1024 + SPECIAL_AUDIO_EOS + assert arch.mask_token_id == 1024 + SPECIAL_AUDIO_MASK + # EOS must remain inside the per-codebook vocab so it stays sampleable. + assert arch.audio_eos_id < arch.num_all_tokens_per_codebook + + +def test_forced_special_token_ids_override_defaults(): + arch = EasyMagpieOmniArch( + codebook_size=1024, + forced_audio_bos_id=1024, + forced_audio_eos_id=1025, + forced_mask_token_id=1028, + ) + assert arch.audio_bos_id == 1024 + assert arch.audio_eos_id == 1025 + assert arch.mask_token_id == 1028 + + +def test_phoneme_ids_fall_back_to_tokenizer_convention(): + arch = EasyMagpieOmniArch(phoneme_vocab_size=2051) + assert arch.resolved_phoneme_bos_id == 2048 + assert arch.resolved_phoneme_eos_id == 2049 + assert arch.resolved_phoneme_unk_id == 2050 + + +def test_from_hf_config_overrides_and_ignores_unknown(): + hf_config = types.SimpleNamespace( + num_audio_codebooks=4, + codebook_size=2048, + frame_stacking_factor=1, + local_transformer_n_layers=5, + some_unrelated_field="ignored", + ) + arch = EasyMagpieOmniArch.from_hf_config(hf_config) + assert arch.num_audio_codebooks == 4 + assert arch.codebook_size == 2048 + assert arch.frame_stacking_factor == 1 + assert arch.local_transformer_n_layers == 5 + # Untouched fields keep the default profile. + assert arch.audio_embedding_dim == EASYMAGPIE_SMALLMAMBA.audio_embedding_dim + + +def test_from_hf_config_hidden_size_fallback(): + hf_config = types.SimpleNamespace(hidden_size=999) + arch = EasyMagpieOmniArch.from_hf_config(hf_config) + assert arch.hidden_dim == 999 + # embedding_dim defaults to the same backbone width when not given explicitly. + assert arch.embedding_dim == 999 diff --git a/examples/tts/easymagpie_vllm_omni/tests/test_local_transformer.py b/examples/tts/easymagpie_vllm_omni/tests/test_local_transformer.py new file mode 100644 index 000000000000..21f9b3f92d98 --- /dev/null +++ b/examples/tts/easymagpie_vllm_omni/tests/test_local_transformer.py @@ -0,0 +1,242 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Validity tests for the vLLM-Omni EasyMagpieTTS local transformer. + +The headline test is a **numerical parity check against the reference NeMo +implementation** (``transformer_2501.Transformer`` + the projection / embedding +heads, exactly as wired in ``EasyMagpieTTSInferenceModel``): random NeMo weights +are copied 1:1 into the vLLM ``EasyMagpieCodePredictor`` and both stacks are run +teacher-forced on identical inputs; the per-codebook logits must match to fp32 +tolerance with identical argmax. This is the pytest port of +``debug_local_transformer.py`` and guards against the re-implementation silently +drifting from the training-time math. + +The remaining tests assert the autoregressive sampler's contract (output shape / +dtype / value range, forbidden-token masking, and seeded determinism). + +Everything runs as plain PyTorch on CPU via the tiny stand-in config from +``conftest.py`` — no model directory, no vLLM engine, no GPU. +""" +from __future__ import annotations + +import pytest + +torch = pytest.importorskip("torch") +pytest.importorskip("vllm") +transformer_2501 = pytest.importorskip("nemo.collections.tts.modules.transformer_2501") + +from conftest import build_vllm_config # noqa: E402 +from easymagpie_vllm_omni.config import EasyMagpieOmniArch # noqa: E402 +from easymagpie_vllm_omni.local_transformer import EasyMagpieCodePredictor # noqa: E402 +from torch import nn # noqa: E402 + +# Two arch profiles: one where all widths are equal (in/out projections are +# Identity, matching the real checkpoint) and one where they differ (projections +# are real Linears) — so the weight-copy + parity covers both code paths. +ARCH_PROFILES = { + "equal_dims": dict( + hidden_dim=64, + embedding_dim=64, + audio_embedding_dim=64, + local_transformer_hidden_dim=64, + local_transformer_n_heads=4, + ), + "mixed_dims": dict( + hidden_dim=64, + embedding_dim=64, + audio_embedding_dim=48, + local_transformer_hidden_dim=80, + local_transformer_n_heads=4, + ), +} + + +class NeMoLocalTransformerStack(nn.Module): + """Reference NeMo local-transformer submodules, named to match the vLLM code predictor. + + Mirrors the wiring in ``EasyMagpieTTSInferenceModel.__init__`` (the + ``local_transformer*`` / ``audio_*`` heads). Attribute names match + :class:`EasyMagpieCodePredictor` so a state-dict copy is 1:1. + """ + + def __init__(self, arch: EasyMagpieOmniArch) -> None: + super().__init__() + self.n_codebooks = arch.num_stacked_codebooks + self.num_all_tokens = arch.num_all_tokens_per_codebook + embedding_dim = arch.embedding_dim + audio_dim = arch.audio_embedding_dim + lt_hidden = arch.local_transformer_hidden_dim + + self.audio_embeddings = nn.ModuleList( + [nn.Embedding(self.num_all_tokens, audio_dim) for _ in range(self.n_codebooks)] + ) + self.audio_in_projection = nn.Linear(audio_dim, embedding_dim) if audio_dim != embedding_dim else nn.Identity() + self.local_transformer_in_projection = ( + nn.Linear(embedding_dim, lt_hidden) if lt_hidden != embedding_dim else nn.Identity() + ) + self.local_transformer = transformer_2501.Transformer( + n_layers=arch.local_transformer_n_layers, + d_model=lt_hidden, + d_ffn=lt_hidden * 4, + sa_n_heads=arch.local_transformer_n_heads, + kernel_size=1, + is_causal=True, + max_length_causal_mask=self.n_codebooks + 2, + use_learnable_pos_emb=True, + ) + self.local_transformer_audio_out_projection = ( + nn.Linear(lt_hidden, audio_dim) if audio_dim != lt_hidden else nn.Identity() + ) + self.local_transformer_out_projections = nn.ModuleList( + [nn.Linear(audio_dim, self.num_all_tokens) for _ in range(self.n_codebooks)] + ) + + @torch.no_grad() + def teacher_forced_logits(self, dec_hidden: torch.Tensor, codes: torch.Tensor) -> torch.Tensor: + """Per-codebook logits given a hidden state and teacher-forced previous codes. + + Replicates ``LocalTransformerHelper.compute_logits`` (AR layout): the input + sequence is ``[dec_hidden, emb(code_0), ..., emb(code_{N-1})]``; row ``k`` of + the causal output predicts codebook ``k``, and the trailing row is dropped. + """ + seq = [dec_hidden] + for k in range(self.n_codebooks): + seq.append(self.audio_in_projection(self.audio_embeddings[k](codes[:, k]))) + x = torch.stack(seq, dim=1) # (T, N+1, embedding_dim) + x = self.local_transformer_in_projection(x) # (T, N+1, lt_hidden) + mask = torch.ones(x.size(0), x.size(1), device=x.device, dtype=x.dtype) + out = self.local_transformer(x, mask)["output"][:, :-1, :] # (T, N, lt_hidden) + out = self.local_transformer_audio_out_projection(out) # (T, N, audio_dim) + logits = [self.local_transformer_out_projections[k](out[:, k, :]) for k in range(self.n_codebooks)] + return torch.stack(logits, dim=1) # (T, N, vocab) + + +@torch.no_grad() +def _vllm_teacher_forced_logits( + cp: EasyMagpieCodePredictor, dec_hidden: torch.Tensor, codes: torch.Tensor +) -> torch.Tensor: + """Per-codebook logits from the vLLM code predictor, teacher-forced. + + Mirrors :meth:`EasyMagpieCodePredictor.generate_codes` buffer layout (``N`` + rows; row 0 = ``in_proj(dec_hidden)``, row ``k+1`` = projected embedding of + ``codes[:, k]``), but reads the logits for every row instead of sampling. + """ + num_tokens = dec_hidden.shape[0] + n = cp.num_codebooks + lt_hidden = cp._buf_inputs.shape[-1] + buf = torch.zeros(num_tokens, n, lt_hidden, dtype=dec_hidden.dtype, device=dec_hidden.device) + buf[:, 0, :] = cp.local_transformer_in_projection(dec_hidden) + for k in range(n - 1): + emb = cp.audio_in_projection(cp.audio_embeddings[k](codes[:, k])) + buf[:, k + 1, :] = cp.local_transformer_in_projection(emb) + hidden = cp.local_transformer(buf) # (T, N, lt_hidden) + logits = [] + for k in range(n): + row = cp.local_transformer_audio_out_projection(hidden[:, k, :]) + logits.append(cp.local_transformer_out_projections[k](row)) + return torch.stack(logits, dim=1) # (T, N, vocab) + + +def _copy_nemo_into_vllm(nemo: NeMoLocalTransformerStack, cp: EasyMagpieCodePredictor) -> None: + """Copy every vLLM code-predictor parameter from the matching NeMo parameter (names align 1:1).""" + nemo_sd = nemo.state_dict() + missing = [] + for name, param in cp.named_parameters(): + if name in nemo_sd: + assert param.shape == nemo_sd[name].shape, f"shape mismatch {name}" + param.data.copy_(nemo_sd[name].to(param.dtype)) + else: + missing.append(name) + assert not missing, f"vLLM params with no NeMo counterpart: {missing}" + + +def _build_pair(profile_kwargs: dict, seed: int = 0): + """Build a (code_predictor, nemo_stack, arch) triple with NeMo weights copied in.""" + cfg = build_vllm_config(**profile_kwargs) + arch = EasyMagpieOmniArch.from_hf_config(cfg.model_config.hf_config) + + cp = EasyMagpieCodePredictor(vllm_config=cfg, prefix="code_predictor").eval() + cp.init_forbidden_mask() + + gen = torch.Generator().manual_seed(seed) + nemo = NeMoLocalTransformerStack(arch).float().eval() + with torch.no_grad(): + for prm in nemo.parameters(): + prm.copy_(torch.empty(prm.shape).normal_(0.0, 0.02, generator=gen)) + _copy_nemo_into_vllm(nemo, cp) + return cp, nemo, arch + + +@pytest.mark.unit +@pytest.mark.parametrize("profile", list(ARCH_PROFILES), ids=list(ARCH_PROFILES)) +def test_local_transformer_matches_nemo(profile): + """vLLM re-implementation must equal the NeMo reference in fp32 (teacher-forced).""" + cp, nemo, arch = _build_pair(ARCH_PROFILES[profile]) + + torch.manual_seed(1234) + num_tokens = 6 + dec_hidden = torch.randn(num_tokens, arch.hidden_dim) + codes = torch.randint(0, arch.codebook_size, (num_tokens, arch.num_stacked_codebooks)) + + nemo_logits = nemo.teacher_forced_logits(dec_hidden, codes) + vllm_logits = _vllm_teacher_forced_logits(cp, dec_hidden, codes) + + assert vllm_logits.shape == nemo_logits.shape + max_abs_diff = (vllm_logits - nemo_logits).abs().max().item() + argmax_mismatch = (vllm_logits.argmax(-1) != nemo_logits.argmax(-1)).sum().item() + assert max_abs_diff < 1e-3, f"max abs diff too large: {max_abs_diff:.3e}" + assert argmax_mismatch == 0, f"{argmax_mismatch} argmax mismatches" + + +@pytest.mark.unit +def test_generate_codes_shape_dtype_and_range(): + """``generate_codes`` returns valid (num_tokens, num_codebooks) int64 codes within vocab.""" + cp, _, arch = _build_pair(ARCH_PROFILES["equal_dims"]) + num_tokens = 5 + + torch.manual_seed(0) + codes = cp.generate_codes(torch.randn(num_tokens, arch.hidden_dim)) + + assert codes.shape == (num_tokens, arch.num_stacked_codebooks) + assert codes.dtype == torch.long + assert codes.min().item() >= 0 + assert codes.max().item() < arch.num_all_tokens_per_codebook + + +@pytest.mark.unit +def test_generate_codes_respects_forbidden_mask(): + """With argmax sampling, forbidden special tokens are never emitted (only EOS stays reachable).""" + cp, _, arch = _build_pair(ARCH_PROFILES["equal_dims"]) + cp.temperature = 0.0 # argmax over masked logits + + torch.manual_seed(0) + codes = cp.generate_codes(torch.randn(7, arch.hidden_dim)) + + # Allowed = real codebook tokens [0, codebook_size) plus the audio EOS id. + allowed = (codes < arch.codebook_size) | (codes == arch.audio_eos_id) + assert allowed.all(), f"sampled forbidden tokens: {sorted(set(codes[~allowed].tolist()))}" + + +@pytest.mark.unit +def test_generate_codes_deterministic_with_seed(): + """Same seed + same input ⇒ identical sampled codes (sampler is RNG-driven, no host state).""" + cp, _, arch = _build_pair(ARCH_PROFILES["equal_dims"]) + dec_hidden = torch.randn(4, arch.hidden_dim) + + torch.manual_seed(7) + first = cp.generate_codes(dec_hidden) + torch.manual_seed(7) + second = cp.generate_codes(dec_hidden) + + assert torch.equal(first, second) From 4eda162dccdf74ba90fc8727813dfd41a3cc22ec Mon Sep 17 00:00:00 2001 From: Viacheslav Klimkov Date: Wed, 3 Jun 2026 16:18:42 +0200 Subject: [PATCH 13/45] examples/tts/easymagpie_vllm_omni: hotfix for nemotron_h in fp16, need scaling Signed-off-by: Viacheslav Klimkov --- .../easymagpie_vllm_omni/backbone_patches.py | 48 +++++++++++++++++++ .../easymagpie_vllm_omni/easymagpie.py | 7 ++- 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/backbone_patches.py b/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/backbone_patches.py index efe8421f7af2..da57ce6742cc 100644 --- a/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/backbone_patches.py +++ b/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/backbone_patches.py @@ -20,6 +20,7 @@ """ from __future__ import annotations +import torch import torch.nn as nn import torch.nn.functional as F from vllm.logger import init_logger @@ -62,3 +63,50 @@ def patch_silu_shared_experts(backbone) -> int: patched += 1 logger.info("SiLU shared_experts fix installed on %d layers", patched) return patched + + +def patch_moe_routed_scale(backbone) -> int: + """Restore ``routed_scaling_factor`` on the NemotronHMoE output in FP16. + + vLLM's ``FusedMoE`` uses an FP16 overflow trick: with + ``apply_routed_scale_to_output=True`` it does **not** multiply the routed + output by ``s`` (=routed_scaling_factor); in FP16 it instead divides the + *shared* output by ``s`` and relies on the decoder layer to keep the whole + residual stream scaled by ``1/s`` (see ``DeepseekV2DecoderLayer.forward``). + NemotronH's decoder layer never applies that compensation, so in FP16 the + MoE block emits ``routed_raw + shared/s == (s*routed + shared)/s`` — the + correct value divided by ``s``. The MoE contribution to the residual ends up + ``s``× too small and the error accumulates across the MoE layers. + + We re-multiply each MoE mixer's output by ``s`` in FP16:: + + s * (routed_raw + shared/s) = s*routed_raw + shared + + which matches the NeMo reference. FP32/BF16 already take the correct + ``fused_output *= s`` branch, so the hook is a no-op there. + + Args: + backbone: the ``NemotronHModel`` instance. + + Returns: + Number of layers patched. + """ + patched = 0 + for layer in backbone.layers: + mixer = getattr(layer, "mixer", None) + if mixer is None or mixer.__class__.__name__ != "NemotronHMoE": + continue + scale = float(getattr(mixer, "routed_scaling_factor", 1.0)) + if scale == 1.0: + continue + + def _scale_output(_mod, _inp, out, _scale=scale): + # FusedMoE only defers the scale in FP16; leave other dtypes alone. + if isinstance(out, torch.Tensor) and out.dtype == torch.float16: + return out * _scale + return out + + mixer.register_forward_hook(_scale_output) + patched += 1 + logger.info("FP16 MoE routed-scale fix installed on %d layers", patched) + return patched diff --git a/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/easymagpie.py b/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/easymagpie.py index d21f357c36aa..184ec50f9e02 100644 --- a/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/easymagpie.py +++ b/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/easymagpie.py @@ -108,7 +108,7 @@ from vllm_omni.model_executor.models.output_templates import OmniOutput -from easymagpie_vllm_omni.backbone_patches import patch_silu_shared_experts +from easymagpie_vllm_omni.backbone_patches import patch_moe_routed_scale, patch_silu_shared_experts from easymagpie_vllm_omni.config import EasyMagpieOmniArch from easymagpie_vllm_omni.local_transformer import EasyMagpieCodePredictor @@ -182,6 +182,11 @@ def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: # NemotronHMLP hard-codes ReLU² in shared_experts. Restore SiLU (no-op # when the backbone has no MoE layers). patch_silu_shared_experts(self.backbone) + # vLLM's FusedMoE defers routed_scaling_factor to the decoder layer in + # FP16, but NemotronH's decoder layer never compensates, so the MoE + # output is under-scaled by routed_scaling_factor. Restore it (no-op in + # fp32/bf16 and when there are no MoE layers). + patch_moe_routed_scale(self.backbone) # ── Local transformer (its own compile group / CUDA graph) ────── with set_model_tag("local_transformer"): From 4c7388ff76b1656f234ca61bb8b6e64611bcbbc6 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimkov Date: Thu, 4 Jun 2026 14:56:35 +0200 Subject: [PATCH 14/45] examples/tts/easymagpie_vllm_omni: introduce EOS forwarding from LT sampled tokens Signed-off-by: Viacheslav Klimkov --- .../benchmark_easymagpie_tts.py | 10 ++- .../easymagpie_inference_demo.ipynb | 35 ++++++--- .../easymagpie_vllm_omni/easymagpie.py | 72 +++++++++++++++++-- 3 files changed, 99 insertions(+), 18 deletions(-) diff --git a/examples/tts/easymagpie_vllm_omni/benchmark_easymagpie_tts.py b/examples/tts/easymagpie_vllm_omni/benchmark_easymagpie_tts.py index c8ac80f4e563..7abe8b296143 100644 --- a/examples/tts/easymagpie_vllm_omni/benchmark_easymagpie_tts.py +++ b/examples/tts/easymagpie_vllm_omni/benchmark_easymagpie_tts.py @@ -246,6 +246,7 @@ class ModelMeta: audio_eos_id: int speech_delay: int frame_stacking_factor: int + stop_token_id: int # backbone stop token the model emits at the audio-EOS frame def _load_model_meta( @@ -302,6 +303,7 @@ def _load_model_meta( audio_eos_id=int(arch.audio_eos_id), speech_delay=int(getattr(arch, "streaming_speech_delay", 0) or 0), frame_stacking_factor=int(arch.frame_stacking_factor), + stop_token_id=EasyMagpieTTSForConditionalGeneration.audio_eos_stop_token_id(type("Cfg", (), config)), ) @@ -462,7 +464,9 @@ async def run_one_request( and isinstance(audio_codes, torch.Tensor) and audio_codes.numel() > 0 ): - if int(audio_codes[-1, 0]) == meta.audio_eos_id: + # audio EOS in ANY codebook (not just codebook 0) — mirrors the + # reference EOS check and the model's own stop signal. + if bool((audio_codes[-1] == meta.audio_eos_id).any()): eos_decode_idx = newest_frame_idx result.eos_reached = True @@ -798,6 +802,10 @@ async def main(args): max_tokens=args.max_new_tokens, detokenize=False, ignore_eos=True, + # The model emits this backbone token at the audio-EOS frame (audio EOS in + # any codebook), so vLLM stops the request there instead of decoding the + # full budget. stop_token_ids is honored even with ignore_eos. + stop_token_ids=[meta.stop_token_id], ) try: diff --git a/examples/tts/easymagpie_vllm_omni/easymagpie_inference_demo.ipynb b/examples/tts/easymagpie_vllm_omni/easymagpie_inference_demo.ipynb index 192e08e1ab03..7aeadfaf7129 100644 --- a/examples/tts/easymagpie_vllm_omni/easymagpie_inference_demo.ipynb +++ b/examples/tts/easymagpie_vllm_omni/easymagpie_inference_demo.ipynb @@ -128,12 +128,15 @@ "TEXT_VOCAB = int(config[\"text_vocab_size\"])\n", "TEXT_EOS_ID = TEXT_VOCAB - 2 # matches EasyMagpieTTSInferenceModel.eos_id\n", "\n", + "AUDIO_STOP_TOKEN_ID = max(1, int(config.get(\"vocab_size\", 2)) - 1)\n", + "\n", "print(f\"Model dir : {MODEL_DIR}\")\n", "print(f\"embedding_dim : {arch.embedding_dim}\")\n", "print(f\"num_stacked_codebooks : {arch.num_stacked_codebooks} (C*S)\")\n", "print(f\"tokens / codebook : {arch.num_all_tokens_per_codebook} (codebook_size + specials)\")\n", "print(f\"audio_bos / audio_eos id : {arch.audio_bos_id} / {arch.audio_eos_id}\")\n", - "print(f\"text_vocab / text_eos : {TEXT_VOCAB} / {TEXT_EOS_ID}\")" + "print(f\"text_vocab / text_eos : {TEXT_VOCAB} / {TEXT_EOS_ID}\")\n", + "print(f\"audio-EOS stop token id : {AUDIO_STOP_TOKEN_ID}\")" ] }, { @@ -216,7 +219,9 @@ " \"temperature\": 0.0,\n", " \"max_tokens\": DECODE_STEPS,\n", " \"detokenize\": False,\n", + " # model forwards EOS to dummy output tokens\n", " \"ignore_eos\": True,\n", + " \"stop_token_ids\": [AUDIO_STOP_TOKEN_ID],\n", " },\n", " }\n", " ],\n", @@ -342,7 +347,12 @@ " temperature=0.0, # backbone token sampler is a no-op (audio is sampled in the local transformer)\n", " max_tokens=DECODE_STEPS,\n", " detokenize=False,\n", - " ignore_eos=True, # audio EOS lives in the codes, not the vLLM token stream -> run the budget + trim\n", + " ignore_eos=True, # audio EOS lives in the codes, not the vLLM token stream\n", + " # The model emits AUDIO_STOP_TOKEN_ID on the backbone stream at the EOS frame\n", + " # (audio EOS in any codebook), so vLLM ends the request there instead of\n", + " # decoding the full DECODE_STEPS budget. stop_token_ids is honored regardless\n", + " # of ignore_eos.\n", + " stop_token_ids=[AUDIO_STOP_TOKEN_ID],\n", ")" ] }, @@ -409,15 +419,20 @@ " print(f\"dropping {speech_delay} leading speech-delay warm-up frames\")\n", " audio_codes = audio_codes[speech_delay:].contiguous()\n", "\n", - " # Trim at the audio EOS: the model signals end-of-speech inside the codes\n", - " # (codebook 0 == audio_eos_id), not via the vLLM token stream.\n", - " eos_frames = (audio_codes[:, 0] == arch.audio_eos_id).nonzero(as_tuple=True)[0]\n", - " if eos_frames.numel() > 0:\n", - " eos_idx = int(eos_frames[0])\n", - " print(f\"audio EOS at frame : {eos_idx} / {audio_codes.shape[0]}\")\n", - " audio_codes = audio_codes[:eos_idx].contiguous()\n", + " # Trim the trailing audio-EOS frame. The engine stops the request the moment\n", + " # the backbone emits AUDIO_STOP_TOKEN_ID (driven high at the audio-EOS frame),\n", + " # so when it finished for that reason the *last* decoded frame is the EOS frame\n", + " # itself — its codes carry audio_eos_id and must not be vocoded.\n", + " # NOTE: we actually expect EOS to be emited\n", + " co = final_ro.outputs[0] if final_ro.outputs else None\n", + " finish_reason = getattr(co, \"finish_reason\", None)\n", + " stop_reason = getattr(co, \"stop_reason\", None)\n", + " print(f\"finish_reason / stop_reason: {finish_reason} / {stop_reason}\")\n", + " if finish_reason == \"stop\" and stop_reason == AUDIO_STOP_TOKEN_ID and audio_codes.shape[0] > 0:\n", + " print(f\"dropping trailing audio-EOS frame at {audio_codes.shape[0] - 1}\")\n", + " audio_codes = audio_codes[:-1].contiguous()\n", " else:\n", - " print(f\"no audio EOS within budget ({DECODE_STEPS} frames); using full decode\")\n", + " print(f\"no engine EOS stop (finish_reason={finish_reason}); using full decode\")\n", "\n", " print(f\"audio_codes shape (decode) : {tuple(audio_codes.shape)}\")\n", " print(f\"audio_codes dtype : {audio_codes.dtype}\")\n", diff --git a/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/easymagpie.py b/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/easymagpie.py index 184ec50f9e02..5c6c29a17d0f 100644 --- a/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/easymagpie.py +++ b/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/easymagpie.py @@ -260,10 +260,36 @@ def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: self._out_codes = torch.zeros(max_num_tokens, self.num_codebooks, dtype=torch.long) + # ── Audio-EOS → engine stop ───────────────────────────────────── + # The model signals end-of-speech inside the audio codebooks. + # To make vLLM terminate the request at the EOS frame, + # we flags decode positions with ``audio_eos_id`` emit designated ``stop_token_id`` + # in ``compute_logits``. + # Callers must pass ``SamplingParams(stop_token_ids=[stop_id])`` with + # ``stop_id = audio_eos_stop_token_id(hf_config)``. + self.audio_eos_id = int(arch.audio_eos_id) + self._stop_token_id = self.audio_eos_stop_token_id(hf_config) + # flags frames in which ``_out_codes`` contain ``audio_eos_id`` + self._token_stop = torch.zeros(max_num_tokens, dtype=torch.bool) + # slice of ``token_stop`` based on ``logit_idx`` that can be used in + # ``compute_logits`` + self._sample_stop = torch.zeros(max_num_tokens, dtype=torch.bool) + # ------------------------------------------------------------------ # Embedding helpers # ------------------------------------------------------------------ + @staticmethod + def audio_eos_stop_token_id(hf_config: Any) -> int: + """Backbone token id this model emits when audio EOS is reached. + + Audio end-of-speech lives in the codebooks, not the backbone token + stream, so the dummy backbone vocab is repurposed as a 2-way stop + signal: index ``0`` == "continue", the last index == "stop". Callers + must pass ``SamplingParams(stop_token_ids=[this])`` + """ + return max(1, int(getattr(hf_config, "vocab_size", 2)) - 1) + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor: """Compatibility shim — unused at runtime (everything goes via inputs_embeds).""" return self.text_embedding(input_ids) @@ -368,7 +394,7 @@ def forward( positions: torch.Tensor, intermediate_tensors: Optional[IntermediateTensors] = None, inputs_embeds: Optional[torch.Tensor] = None, - **_: Any, + **kwargs: Any, ) -> torch.Tensor: """Assemble the per-token embedding, run the backbone, then the codes. @@ -385,6 +411,11 @@ def forward( else: combined.zero_() + # Reset per-token stop flags for this step (so prefill / warm-up rows stay + # "continue"); decode positions get set below by :meth:`_flag_audio_eos`. + self._token_stop[:num_tokens].zero_() + logits_index = kwargs.get("logits_index") + decode_idx, num_req = self._get_decode_idxs() if decode_idx is None: @@ -406,6 +437,7 @@ def forward( if decode_idx is None: codes = self.code_predictor.generate_codes(hidden_states) self._out_codes[:num_tokens].copy_(codes) + self._flag_audio_eos(codes, slice(0, num_tokens)) if self.has_phoneme: self._predict_phonemes(hidden_states, slice(0, num_tokens)) elif num_req > 0: @@ -416,11 +448,30 @@ def forward( ctx.batch_descriptor = orig_bd valid = decode_idx[:num_req] self._out_codes[valid] = codes[:num_req] + self._flag_audio_eos(codes[:num_req], valid) if self.has_phoneme: self._predict_phonemes(hidden_states, valid) + # Re-index _token_stop into _sample_stop. + # this only happens for mixed/prefill, since for capture logits_index is None, + # so during decode-only the branch for logits_index is None will be executed. + if logits_index is not None: + self._sample_stop[:logits_index.shape[0]] = self._token_stop[logits_index] + else: + self._sample_stop[:num_tokens].copy_(self._token_stop[:num_tokens]) + return hidden_states + def _flag_audio_eos(self, codes: torch.Tensor, idx) -> None: + """Flag decode positions whose newly sampled frame ends speech. + Checks codes for eos and assigns token_stop[idx] + + Note: this uses the *sampled* codes. NeMo also checks armax(logits) == eos_idx, + i.e. checks if EOS is emited without sampling. Skip for now. + """ + eos = (codes == self.audio_eos_id).any(dim=1) & (self._dec_audio_valid[idx] == 1) + self._token_stop[idx] = eos + def _assemble_decode_embeddings(self, combined: torch.Tensor, idx) -> None: """Add ``text + phoneme + audio`` embeddings into ``combined`` at ``idx``.""" # Audio: previous-frame codes (gated by validity). @@ -477,18 +528,25 @@ def _predict_phonemes(self, hidden_states: torch.Tensor, idx) -> None: # ------------------------------------------------------------------ def compute_logits(self, hidden_states, sampling_metadata: Any = None) -> Optional[torch.Tensor]: - """Return zero logits so vLLM's sampler always picks index 0. - - The width is taken from ``hf_config.vocab_size`` so the sampler's working - buffers match. The sampled id is irrelevant — audio is surfaced via - :meth:`make_omni_output`. + f"""Dummy backbone logits, repurposed as a 2-way continue/stop signal. + ``_sample_stop`` indicates which frames contain EOS. We set logits, + based on that: logits[sample_stop == True, stop_token_id] = 30 or -30 otherwise. + SamplingParams should set stop_token_id as EOS token though. """ if isinstance(hidden_states, OmniOutput): hidden_states = hidden_states.text_hidden_states if hidden_states is None: return None batch_size = hidden_states.shape[0] - return hidden_states.new_zeros(batch_size, int(self.hf_config.vocab_size)) + logits = hidden_states.new_zeros(batch_size, int(self.hf_config.vocab_size)) + if self._stop_token_id < logits.shape[1]: + stop_rows = self._sample_stop[:batch_size] + logits[:, self._stop_token_id] = torch.where( + stop_rows, + logits.new_full((), 30.0), + logits.new_full((), -30.0), + ) + return logits # ------------------------------------------------------------------ # multimodal output plumbing From 42105350988dab8068b63c35047e1330f5496df2 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimkov Date: Thu, 4 Jun 2026 18:47:03 +0200 Subject: [PATCH 15/45] examples/tts/easymagpie_vllm_omni: initial version of TTS service Signed-off-by: Viacheslav Klimkov --- examples/tts/easymagpie_vllm_omni/Dockerfile | 23 + .../export_codec_decoder_onnx.py | 283 ++++++++++++ .../export_codec_decoder_trt.py | 119 +++++ .../model_repository/codec/config.pbtxt | 33 ++ .../model_repository/easymp/1/model.py | 420 ++++++++++++++++++ .../model_repository/easymp/config.pbtxt | 103 +++++ 6 files changed, 981 insertions(+) create mode 100644 examples/tts/easymagpie_vllm_omni/Dockerfile create mode 100644 examples/tts/easymagpie_vllm_omni/export_codec_decoder_onnx.py create mode 100644 examples/tts/easymagpie_vllm_omni/export_codec_decoder_trt.py create mode 100644 examples/tts/easymagpie_vllm_omni/model_repository/codec/config.pbtxt create mode 100644 examples/tts/easymagpie_vllm_omni/model_repository/easymp/1/model.py create mode 100644 examples/tts/easymagpie_vllm_omni/model_repository/easymp/config.pbtxt diff --git a/examples/tts/easymagpie_vllm_omni/Dockerfile b/examples/tts/easymagpie_vllm_omni/Dockerfile new file mode 100644 index 000000000000..f9b29c757618 --- /dev/null +++ b/examples/tts/easymagpie_vllm_omni/Dockerfile @@ -0,0 +1,23 @@ +FROM nvcr.io/nvidia/tritonserver:26.02-py3 + +# 1. System dependency for git-based installs +RUN apt-get update && \ + apt-get install -y git sox libsox-fmt-all + +# 2. upstream vllm +RUN pip install --no-cache-dir \ + "vllm==0.21.0" \ + "vllm_omni==0.21.0rc1" + +# 3. TODO install NeMo/examples/tts/easymagpie_vllm_omni + +# 4. Extra python requirements needed to compile the model +RUN pip install --no-cache-dir \ + onnxscript \ + librosa \ + sox \ + onnx-graphsurgeon \ + "tritonclient[grpc]" +RUN pip install --no-cache-dir --force-reinstall --no-deps "numpy==2.3.5" + +WORKDIR /workspace diff --git a/examples/tts/easymagpie_vllm_omni/export_codec_decoder_onnx.py b/examples/tts/easymagpie_vllm_omni/export_codec_decoder_onnx.py new file mode 100644 index 000000000000..85098993daad --- /dev/null +++ b/examples/tts/easymagpie_vllm_omni/export_codec_decoder_onnx.py @@ -0,0 +1,283 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Stage 1/2: export the EasyMagpieTTS (25 fps spectral) codec decoder to ONNX. + +The exported graph takes the model's **raw stacked codes** and produces a +waveform, baking the two stateless model->codec glue steps into the graph so the +serving side (e.g. a Triton python backend) needs no NeMo: + + input audio_codes : int64 (batch, frames, num_stacked_codebooks) + output audio_values : float (batch, frames * output_samples_per_frame) + + audio_codes -> clamp(specials) -> unstack -> index-convert -> codec.decode + +``num_stacked_codebooks = num_audio_codebooks * frame_stacking_factor`` (e.g. +``8 * 2 = 16``). With ``--nemo_file`` the wrapper: + +* **clamps** out-of-range special tokens (audio bos/eos/mask) to valid indices, +* **unstacks** ``(B, T, C*S) -> (B, C, T*S)`` (inverse of ``stack_codes``), and +* **index-converts** the model's regrouped FSQ space (e.g. 8 codebooks of 1024) + to the codec's native ``GroupFiniteScalarQuantizer`` space (e.g. 5 codebooks of + 4^8) via ``VectorQuantizerIndexConverter.convert_new_to_original`` -- a lossless + per-frame index remap, read straight from the EasyMagpie ``.nemo``. + +Without ``--nemo_file`` it falls back to the codec's *native* decode (input +``(batch, frames, num_codebooks)``, no unstack / convert). + +For ``25fps_spectral_codec_with_bandwidth_extension.nemo`` the codec emits 882 +output samples / frame (decode emits 22050 Hz; encoder runs at 16000 Hz / 640 +samples per frame); one model frame unstacks to ``frame_stacking_factor`` codec +frames. + +The frame axis is exported as **static** (``--frames``) and only ``batch`` is +dynamic -- this matches the streaming decode usage (a fixed chunk size) and lets +TensorRT pick efficient tactics. Build several engines if you need several chunk +sizes, or pass a frames profile to the TRT builder for a dynamic frame axis. + +Stage 2 (TRT engine build) lives in ``export_codec_decoder_trt.py``. + +Example: + python examples/tts/easymagpie_vllm_omni/export_codec_decoder_onnx.py \\ + --codec_model_path /path/to/25fps_spectral_codec_with_bandwidth_extension.nemo \\ + --nemo_file /path/to/easymagpie.nemo \\ + --onnx-path codec/codec_decoder.onnx \\ + --frames 15 --device cuda +""" +from __future__ import annotations + +import argparse +from pathlib import Path + +import numpy as np +import torch + +# Match ORT's full-FP32 matmul; PyTorch on Ampere+ uses TF32 by default and would +# otherwise diverge from the ONNX/ORT reference during the parity check. +torch.backends.cuda.matmul.allow_tf32 = False +torch.backends.cudnn.allow_tf32 = False + +try: + import onnx +except ImportError as exc: # pragma: no cover + raise ImportError("`onnx` is required. Install with: pip install onnx onnxruntime") from exc + +from nemo.collections.tts.models import AudioCodecModel +from nemo.utils import logging + + +class CodecDecoderWrapper(torch.nn.Module): + """Wrap ``AudioCodecModel`` so a single ``(B, T, C)`` int tensor decodes to ``(B, T_audio)``. + + With ``converter``/``stacking`` set, the input is the model's *stacked* codes + ``(B, T, C*S)`` and the wrapper clamps special tokens, unstacks to ``(B, C, T*S)`` + and index-converts to the codec's native space before decoding. Otherwise the + input is the codec's *native* codes ``(B, T, num_codebooks)``. + + The codec's conv layers mask out-of-range positions using a per-batch length. + We bake a *full-length* length tensor (all frames valid) so the mask folds to a + constant at export time and disappears from the graph. + """ + + def __init__( + self, + codec_model: AudioCodecModel, + converter: torch.nn.Module = None, + stacking: int = 1, + clamp_max: int = None, + ): + super().__init__() + self.codec_model = codec_model + self.converter = converter + self.stacking = int(stacking) + self.clamp_max = clamp_max + + def forward(self, audio_codes: torch.Tensor) -> torch.Tensor: + # audio_codes: (B, T, C) -> codec expects (B, C, T) + tokens = audio_codes.transpose(1, 2).contiguous() + bsz = tokens.shape[0] + + if self.stacking > 1: + # Unstack (B, C*S, T) -> (B, C, T*S): inverse of EasyMagpie stack_codes. + cs, t = tokens.shape[1], tokens.shape[2] + c = cs // self.stacking + tokens = tokens.view(bsz, c, self.stacking, t).permute(0, 1, 3, 2).reshape(bsz, c, t * self.stacking) + + if self.clamp_max is not None: + # Drop special tokens (audio bos/eos/mask live above the codebook). + tokens = tokens.clamp(0, self.clamp_max) + + tokens = tokens.contiguous() + frames = tokens.shape[2] + tokens_len = torch.full((bsz,), frames, dtype=torch.long, device=tokens.device) + + if self.converter is not None: + tokens = self.converter.convert_new_to_original(audio_tokens=tokens, audio_lens=tokens_len) + + audio, _ = self.codec_model.decode(tokens=tokens, tokens_len=tokens_len) + return audio + + +def check_onnx_parity(wrapper, onnx_path, audio_codes, device, atol=1e-3): + try: + import onnxruntime as ort + except ImportError: + logging.warning("onnxruntime not installed -- skipping parity check") + return True + + providers = ( + ["CUDAExecutionProvider", "CPUExecutionProvider"] if device.type == "cuda" else ["CPUExecutionProvider"] + ) + sess = ort.InferenceSession(str(onnx_path), providers=providers) + + with torch.inference_mode(): + ref = wrapper(audio_codes).detach().cpu().float().numpy() + ort_out = sess.run(None, {"audio_codes": audio_codes.cpu().numpy()})[0] + max_diff = float(np.abs(ref - ort_out).max()) + ok = max_diff <= atol + logging.info( + f"ONNX parity ({sess.get_providers()[0]}): max_abs_diff={max_diff:.6f} " + f"atol={atol} {'PASSED' if ok else 'FAILED'}" + ) + return ok + + +def load_codec_decoder(codec_model_path: str, device: torch.device) -> AudioCodecModel: + """Restore the codec in FP32/eval and strip the (unused at inference) discriminator.""" + codec_cfg = AudioCodecModel.restore_from(codec_model_path, return_config=True) + if "use_scl_loss" in codec_cfg: + codec_cfg.use_scl_loss = False + codec = AudioCodecModel.restore_from(codec_model_path, strict=False, override_config_path=codec_cfg) + if hasattr(codec, "discriminator"): + del codec.discriminator + codec = codec.to(device).eval().float() + codec.freeze() + # Fuse weight-norm reparameterizations into plain conv weights for a clean graph. + if hasattr(codec, "audio_decoder") and hasattr(codec.audio_decoder, "remove_weight_norm"): + codec.audio_decoder.remove_weight_norm() + return codec + + +def load_index_converter(codec: AudioCodecModel, nemo_file: str, device: torch.device): + """Build the model->codec index converter + stacking factor from an EasyMagpie .nemo. + + Reads only the EasyMagpie config (no weights): the ``vector_quantizer`` override + the model was trained with and its ``frame_stacking_factor``. Returns + ``(converter_or_None, stacking, new_codebook_size)``. ``converter`` is None when + the model and codec already share the same FSQ grouping. + """ + from hydra.utils import instantiate + + from nemo.collections.tts.models.easy_magpietts_inference import EasyMagpieTTSInferenceModel + from nemo.collections.tts.modules.audio_codec_modules import VectorQuantizerIndexConverter + + em_cfg = EasyMagpieTTSInferenceModel.restore_from(nemo_file, return_config=True) + stacking = int(em_cfg.get("frame_stacking_factor", 1)) + vq_cfg = em_cfg.get("vector_quantizer") + if vq_cfg is None: + return None, stacking, None + + vq_new = instantiate(vq_cfg).to(device).eval() + new_codebook_size = int(vq_new.codebook_size) + if vq_new.num_codebooks == codec.vector_quantizer.num_codebooks: + return None, stacking, new_codebook_size + + converter = VectorQuantizerIndexConverter( + vector_quantizer_original=codec.vector_quantizer, + vector_quantizer_new=vq_new, + ).to(device).eval() + return converter, stacking, new_codebook_size + + +def parse_args(): + p = argparse.ArgumentParser(description="Export the EasyMagpieTTS codec decoder to ONNX") + p.add_argument("--codec_model_path", required=True, help="Path to the audio codec .nemo checkpoint") + p.add_argument( + "--nemo_file", + default=None, + help="EasyMagpie .nemo: bakes unstack + index conversion in (input becomes stacked model codes). " + "Omit to export the codec's native decode.", + ) + p.add_argument("--onnx-path", default="codec_decoder.onnx") + p.add_argument("--frames", type=int, default=30, help="Static frame count baked into the graph (chunk size)") + p.add_argument("--batch-size", type=int, default=1, help="Dummy batch size used for export/parity") + p.add_argument("--opset", type=int, default=18) + p.add_argument("--device", default="cpu", choices=["cpu", "cuda"]) + p.add_argument("--atol", type=float, default=1e-3) + return p.parse_args() + + +def main(): + args = parse_args() + device = torch.device(args.device) + + codec = load_codec_decoder(args.codec_model_path, device) + + converter, stacking, new_codebook_size = (None, 1, None) + if args.nemo_file is not None: + converter, stacking, new_codebook_size = load_index_converter(codec, args.nemo_file, device) + + if args.nemo_file is not None: + # Input is the model's stacked codes; clamp specials, unstack, convert. + model_codebooks = ( + converter.vector_quantizer_new.num_codebooks if converter is not None else int(codec.num_codebooks) + ) + codebook_size = new_codebook_size if new_codebook_size is not None else int(codec.codebook_size) + nq = model_codebooks * stacking # num_stacked_codebooks (e.g. 16) + clamp_max = codebook_size - 1 + else: + # Input is the codec's native codes. + nq = int(codec.num_codebooks) + codebook_size = int(codec.codebook_size) + clamp_max = None + + wrapper = CodecDecoderWrapper(codec, converter=converter, stacking=stacking, clamp_max=clamp_max).to(device).eval() + + logging.info( + f"codec: sample_rate={codec.sample_rate} output_sample_rate={codec.output_sample_rate} " + f"samples_per_frame={codec.samples_per_frame} native_codebooks={int(codec.num_codebooks)} " + f"| input num_codebooks={nq} stacking={stacking} convert={converter is not None}" + ) + + dummy = torch.randint(0, codebook_size, (args.batch_size, args.frames, nq), dtype=torch.long, device=device) + + onnx_path = Path(args.onnx_path) + onnx_path.parent.mkdir(parents=True, exist_ok=True) + + with torch.inference_mode(): + torch.onnx.export( + wrapper, + (dummy,), + str(onnx_path), + dynamo=False, + export_params=True, + opset_version=args.opset, + do_constant_folding=True, + input_names=["audio_codes"], + output_names=["audio_values"], + dynamic_axes={ + "audio_codes": {0: "batch"}, + "audio_values": {0: "batch"}, + }, + ) + logging.info(f"ONNX exported to {onnx_path}") + + onnx.checker.check_model(str(onnx_path)) + + if not check_onnx_parity(wrapper, onnx_path, dummy, device, atol=args.atol): + raise RuntimeError("ONNX vs PyTorch parity failed -- export is broken.") + + +if __name__ == "__main__": + main() diff --git a/examples/tts/easymagpie_vllm_omni/export_codec_decoder_trt.py b/examples/tts/easymagpie_vllm_omni/export_codec_decoder_trt.py new file mode 100644 index 000000000000..67d2b0174107 --- /dev/null +++ b/examples/tts/easymagpie_vllm_omni/export_codec_decoder_trt.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Stage 2/2: build a TensorRT engine from the codec-decoder ONNX. + +Consumes the ONNX produced by ``export_codec_decoder_onnx.py`` and runs +``trtexec`` to build an engine with a dynamic ``batch`` (and optionally dynamic +``frames``) shape profile. + +The input tensor is ``audio_codes`` with shape ``(batch, frames, num_codebooks)``. +``num_codebooks`` is read from the ONNX graph; ``batch``/``frames`` come from the +profile flags. + +Example: + python examples/tts/easymagpie_vllm_omni/export_codec_decoder_trt.py \\ + --onnx-path codec/codec_decoder.onnx \\ + --trt-path codec/codec_decoder.plan \\ + --batch-profile 1 8 32 \\ + --frames-profile 30 30 30 --fp16 + +Notes +----- +* The frame axis is usually static (export with a fixed ``--frames`` and use the + same value for min/opt/max). A dynamic frame axis works too if the ONNX was + exported with ``frames`` dynamic. +""" +from __future__ import annotations + +import argparse +import shutil +import subprocess +from pathlib import Path + +import onnx + + +def _infer_num_quantizers(onnx_path): + model = onnx.load(str(onnx_path)) + for inp in model.graph.input: + if inp.name != "audio_codes": + continue + dims = inp.type.tensor_type.shape.dim + if len(dims) >= 3 and dims[2].dim_value > 0: + return int(dims[2].dim_value) + raise RuntimeError( + f"could not infer num_quantizers from {onnx_path} (audio_codes dim 2 is not a static positive integer)" + ) + + +def convert_to_trt(onnx_path, trt_path, trtexec_bin, nq, batch_prof, frames_prof, fp32): + exe = shutil.which(trtexec_bin) if "/" not in trtexec_bin else trtexec_bin + if exe is None: + raise FileNotFoundError(f"trtexec not found: {trtexec_bin}") + trt_path.parent.mkdir(parents=True, exist_ok=True) + + def s(b, f): + return f"{b}x{f}x{nq}" + + cmd = [ + exe, + f"--onnx={onnx_path}", + f"--saveEngine={trt_path}", + f"--minShapes=audio_codes:{s(batch_prof[0], frames_prof[0])}", + f"--optShapes=audio_codes:{s(batch_prof[1], frames_prof[1])}", + f"--maxShapes=audio_codes:{s(batch_prof[2], frames_prof[2])}", + ] + if not fp32: + cmd.append("--fp16") + print("Running:", " ".join(cmd)) + subprocess.run(cmd, check=True) + print(f"TensorRT engine saved to {trt_path}") + + +def parse_args(): + p = argparse.ArgumentParser(description="Build a TensorRT engine from the codec-decoder ONNX") + p.add_argument("--onnx-path", required=True) + p.add_argument("--trt-path", required=True) + p.add_argument("--trtexec-bin", default="/usr/src/tensorrt/bin/trtexec") + p.add_argument("--batch-profile", nargs=3, type=int, default=[1, 8, 32], metavar=("MIN", "OPT", "MAX")) + p.add_argument("--frames-profile", nargs=3, type=int, default=[15, 15, 15], metavar=("MIN", "OPT", "MAX")) + p.add_argument("--fp32", action="store_true", help="Build pure FP32 engine (default: FP16).") + return p.parse_args() + + +def main(): + args = parse_args() + onnx_path = Path(args.onnx_path) + trt_path = Path(args.trt_path) + + if not onnx_path.is_file(): + raise FileNotFoundError(f"ONNX not found: {onnx_path}") + + nq = _infer_num_quantizers(onnx_path) + print(f"num_quantizers={nq} (from {onnx_path})") + + convert_to_trt( + onnx_path, + trt_path, + args.trtexec_bin, + nq=nq, + batch_prof=tuple(args.batch_profile), + frames_prof=tuple(args.frames_profile), + fp32=args.fp32, + ) + + +if __name__ == "__main__": + main() diff --git a/examples/tts/easymagpie_vllm_omni/model_repository/codec/config.pbtxt b/examples/tts/easymagpie_vllm_omni/model_repository/codec/config.pbtxt new file mode 100644 index 000000000000..a575964c4dba --- /dev/null +++ b/examples/tts/easymagpie_vllm_omni/model_repository/codec/config.pbtxt @@ -0,0 +1,33 @@ +name: "codec" +platform: "tensorrt_plan" +max_batch_size: 32 + +# Stacked model codes (clamp + unstack + index-convert are baked into the engine). +# Frame axis is static at the exported chunk size (15 model frames). +input [ + { + name: "audio_codes" + data_type: TYPE_INT64 + dims: [ 15, 16 ] + } +] + +output [ + { + name: "audio_values" + data_type: TYPE_FP32 + dims: [ -1 ] + } +] + +dynamic_batching { + max_queue_delay_microseconds: 1000 + preferred_batch_size: [ 32 ] +} + +instance_group [ + { + count: 1 + kind: KIND_GPU + } +] diff --git a/examples/tts/easymagpie_vllm_omni/model_repository/easymp/1/model.py b/examples/tts/easymagpie_vllm_omni/model_repository/easymp/1/model.py new file mode 100644 index 000000000000..237a7125e5ee --- /dev/null +++ b/examples/tts/easymagpie_vllm_omni/model_repository/easymp/1/model.py @@ -0,0 +1,420 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Triton Python backend for EasyMagpieTTS driven by vllm-omni's AsyncOmni engine. + +Wraps ``EasyMagpieTTSForConditionalGeneration`` (the vLLM-Omni talker, same model +used by the inference demo / benchmark): it streams stacked codec frames, which we +chunk-decode (overlap-save) through the ``codec`` TensorRT model. + +Pipeline: + 1. Build ``additional_information`` from ``{speaker_embedding, context_text, text, + temperature, top_k}`` and a placeholder ``prompt_token_ids`` of length + ``estimate_prompt_len(...)``. + 2. Submit one request to ``AsyncOmni.generate()``. Each step yields the + *cumulative* ``audio_codes`` tensor ``(T_total, C*S)`` (prefill rows + one row + per decode step) and cumulative backbone ``token_ids``; we slice the decoded + rows, drop the leading ``speech_delay`` warm-up frames, and stop at the audio + EOS frame. + 3. New frames are streamed out in fixed ``codec_chunk_size``-frame windows (with a + trimmed ``codec_left_context``) through the ``codec`` BLS, which unstacks + + index-converts + decodes them to 22.05 kHz audio chunks. +""" + +from __future__ import annotations + +import asyncio +import concurrent.futures +import json +import logging +import os +import queue +import tempfile +import threading +import time +import uuid +from pathlib import Path + +import numpy as np +import torch +import triton_python_backend_utils as pb_utils +import yaml + +logging.basicConfig( + format="%(asctime)s [%(levelname)s]: %(message)s", + level=logging.INFO, + datefmt="%Y-%m-%d %H:%M:%S", +) +logger = logging.getLogger("easymp_triton") + + +def _require_param(parameters: dict, key: str) -> str: + val = parameters.get(key) + if isinstance(val, dict): + val = val.get("string_value") + if val is None: + raise KeyError(f"Missing required model parameter: {key!r}") + return str(val) + + +class TritonPythonModel: + def initialize(self, args): + os.environ.setdefault("VLLM_WORKER_MULTIPROC_METHOD", "spawn") + + self.model_config = json.loads(args["model_config"]) + params = self.model_config.get("parameters", {}) + + self.vllm_model_path = _require_param(params, "vllm_model_path") + self.default_speaker = _require_param(params, "default_speaker") + self.default_context_text = _require_param(params, "default_context_text") + + self.max_model_len = int(_require_param(params, "max_model_len")) + self.max_num_seqs = int(_require_param(params, "max_num_seqs")) + self.max_num_batched_tokens = int(_require_param(params, "max_num_batched_tokens")) + self.max_new_tokens = int(_require_param(params, "max_new_tokens")) + self.gpu_memory_utilization = float(_require_param(params, "gpu_memory_utilization")) + + self.codec_chunk_size = int(_require_param(params, "codec_chunk_size")) + self.codec_left_context = int(_require_param(params, "codec_left_context")) + self.first_chunk_frames = int(_require_param(params, "first_chunk_frames")) + + self.lt_temperature = float(_require_param(params, "lt_temperature")) + self.lt_top_k = int(_require_param(params, "lt_top_k")) + + self._load_arch_and_tokenizer() + self._speaker_cache: dict = {} + # Inferred from the first codec decode (audio_len / codec_chunk_size). + self._spf: int | None = None + + self._loop = asyncio.new_event_loop() + self._loop_thread = threading.Thread(target=self._loop.run_forever, daemon=True) + self._loop_thread.start() + + # One thread per in-flight request serializes its codec decode + + # response_sender.send calls, off the asyncio loop and overlapping with + # vLLM generation; Triton dynamic batching then groups the codec calls. + self._codec_pool = concurrent.futures.ThreadPoolExecutor( + max_workers=max(1, self.max_num_seqs), + thread_name_prefix="easymp_codec", + ) + + self._start_omni_engine() + logger.info("EasyMagpie initialized (default_speaker=%s)", self.default_speaker) + + def _load_arch_and_tokenizer(self): + from transformers import AutoTokenizer + + from easymagpie_vllm_omni.config import EasyMagpieOmniArch + from easymagpie_vllm_omni.easymagpie import EasyMagpieTTSForConditionalGeneration + + config = json.loads((Path(self.vllm_model_path) / "config.json").read_text()) + cfg_obj = type("Cfg", (), config) + arch = EasyMagpieOmniArch.from_hf_config(cfg_obj) + + self.audio_eos_id = int(arch.audio_eos_id) + self.speech_delay = int(getattr(arch, "streaming_speech_delay", 0) or 0) + self.num_stacked_codebooks = int(arch.num_stacked_codebooks) + self.has_task_embedding = arch.num_task_embeddings > 0 + self.stop_token_id = EasyMagpieTTSForConditionalGeneration.audio_eos_stop_token_id(cfg_obj) + + self.tokenizer = AutoTokenizer.from_pretrained(self.vllm_model_path, trust_remote_code=True) + self._estimate_prompt_len = EasyMagpieTTSForConditionalGeneration.estimate_prompt_len + + def _build_stage_config_file(self) -> str: + stage_cfg = { + "stage_args": [ + { + "stage_id": 0, + "stage_type": "llm", + "is_comprehension": True, + "final_output": True, + "final_output_type": "audio", + "runtime": {"devices": "0"}, + "engine_args": { + "model_stage": "easymagpie", + "max_num_seqs": self.max_num_seqs, + "model_arch": "EasyMagpieTTSForConditionalGeneration", + "worker_type": "ar", + "scheduler_cls": "vllm_omni.core.sched.omni_ar_scheduler.OmniARAsyncScheduler", + "enforce_eager": False, + "trust_remote_code": True, + "async_scheduling": True, + "enable_prefix_caching": False, + "engine_output_type": "audio", + "gpu_memory_utilization": self.gpu_memory_utilization, + "distributed_executor_backend": "uni", + "max_num_batched_tokens": self.max_num_batched_tokens, + "max_model_len": self.max_model_len, + # bf16 overflows the Nemotron-H fused-MoE Triton kernel's + # fp32 shared memory; fp16 backbone + fp32 mamba cache. + "dtype": "float16", + "mamba_ssm_cache_dtype": "float32", + "attention_backend": "TRITON_ATTN", + # We feed prompt_token_ids directly; the model loads the + # bundled tokenizer to tokenize context_text + text. + "skip_tokenizer_init": True, + }, + "default_sampling_params": { + # Backbone token sampler is a no-op (audio is sampled in the + # local transformer via additional_information temperature/top_k). + "temperature": 0.0, + "max_tokens": self.max_new_tokens, + "detokenize": False, + # Audio EOS lives in the codes; the model emits stop_token_id + # on the backbone stream at the EOS frame. + "ignore_eos": True, + "stop_token_ids": [self.stop_token_id], + }, + } + ], + } + tmp = tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", prefix="easymp_triton_", delete=False) + yaml.dump(stage_cfg, tmp, sort_keys=False) + tmp.close() + return tmp.name + + def _start_omni_engine(self): + from vllm_omni import AsyncOmni + + self._stage_cfg_path = self._build_stage_config_file() + self.omni = AsyncOmni( + model=self.vllm_model_path, + stage_configs_path=self._stage_cfg_path, + log_stats=False, + stage_init_timeout=300, + ) + + def _get_speaker_embedding(self, speaker: str) -> torch.Tensor: + if speaker not in self._speaker_cache: + emb_path = Path(self.vllm_model_path) / "speaker_embeddings" / f"{speaker}.pt" + if not emb_path.exists(): + raise FileNotFoundError(f"Speaker embedding not found: {emb_path}") + loaded = torch.load(emb_path, map_location="cpu") + emb = loaded["speaker_encoding"] if isinstance(loaded, dict) else loaded + self._speaker_cache[speaker] = emb.to(torch.float32) + return self._speaker_cache[speaker] + + def _build_prompt(self, text: str, context_text: str, speaker: str) -> dict: + speaker_embedding = self._get_speaker_embedding(speaker) + prompt_len = self._estimate_prompt_len( + speaker_embedding, + tokenize=lambda t: self.tokenizer.encode(t), + context_text=context_text, + has_task_embedding=self.has_task_embedding, + ) + return { + "prompt_token_ids": [0] * prompt_len, + "additional_information": { + "speaker_embedding": speaker_embedding, + "context_text": context_text, + "text": text, + "temperature": self.lt_temperature, + "top_k": self.lt_top_k, + }, + } + + def _decode_codec(self, codes: torch.Tensor, left_context_frames: int) -> np.ndarray: + """Decode one ``(<=codec_chunk_size, C*S)`` window, trim left context + pad.""" + codes_np = codes.detach().cpu().to(torch.int64).numpy() + pad = self.codec_chunk_size - codes_np.shape[0] + if pad > 0: + codes_np = np.pad(codes_np, ((0, pad), (0, 0))) + + response = pb_utils.InferenceRequest( + model_name="codec", + requested_output_names=["audio_values"], + inputs=[pb_utils.Tensor("audio_codes", codes_np[np.newaxis])], + ).exec() + if response.has_error(): + raise RuntimeError(f"Codec decode failed: {response.error().message()}") + + audio_tensor = pb_utils.get_output_tensor_by_name(response, "audio_values") + audio = ( + audio_tensor.as_numpy() + if audio_tensor.is_cpu() + else torch.from_dlpack(audio_tensor.to_dlpack()).cpu().numpy() + ) + if audio.ndim > 1: + audio = audio[0] + + if self._spf is None: + self._spf = audio.shape[-1] // self.codec_chunk_size + left = left_context_frames * self._spf + right = pad * self._spf + return audio[left:-right] if right > 0 else audio[left:] + + def _send_audio(self, response_sender, audio: np.ndarray, final: bool): + response_sender.send( + pb_utils.InferenceResponse(output_tensors=[pb_utils.Tensor("audio", audio.astype(np.float32))]), + flags=pb_utils.TRITONSERVER_RESPONSE_COMPLETE_FINAL if final else 0, + ) + + def _send_error(self, response_sender, err: Exception): + try: + response_sender.send( + pb_utils.InferenceResponse(output_tensors=[], error=pb_utils.TritonError(str(err))), + flags=pb_utils.TRITONSERVER_RESPONSE_COMPLETE_FINAL, + ) + except Exception: + pass + + def _codec_worker(self, codec_q: queue.Queue, response_sender, state: dict) -> None: + """Pop ``(chunk, ctx, is_final)`` tuples; ``None`` == send empty final + exit.""" + finalized = False + try: + while True: + item = codec_q.get() + if item is None: + self._send_audio(response_sender, np.array([], dtype=np.float32), final=True) + finalized = True + return + chunk, ctx, is_final = item + audio = self._decode_codec(chunk, ctx) + self._send_audio(response_sender, audio, final=is_final) + if state["t_first_audio"] is None: + state["t_first_audio"] = time.perf_counter() + if is_final: + finalized = True + return + except Exception as e: + state["error"] = e + if not finalized: + self._send_error(response_sender, e) + + async def _synthesize(self, text: str, context_text: str, speaker: str, response_sender): + t_start = time.perf_counter() + request_id = f"easymp-{uuid.uuid4().hex[:8]}" + prompt = self._build_prompt(text, context_text, speaker) + prompt_len = len(prompt["prompt_token_ids"]) + + codec_q: queue.Queue = queue.Queue() + state: dict = {"t_first_audio": None, "error": None} + codec_future = self._codec_pool.submit(self._codec_worker, codec_q, response_sender, state) + + L = self.codec_left_context + sent = 0 # real frames (post speech-delay, pre EOS) already queued + threshold = self.first_chunk_frames + real: torch.Tensor | None = None + real_count = 0 + eos_found = False + + try: + async for out in self.omni.generate(prompt, request_id=request_id): + if state["error"] is not None: + break + mm = getattr(out, "multimodal_output", None) or {} + audio_codes = mm.get("audio_codes") + if not isinstance(audio_codes, torch.Tensor): + continue + # audio_codes accumulates one row per flat-batch token: prompt_len + # prefill rows + one per decode step. Count decoded frames from the + # tensor (token_ids on a streaming step is a delta, not cumulative). + num_decoded = audio_codes.shape[0] - prompt_len + if num_decoded <= self.speech_delay: + continue + + # Decoded rows are everything after prefill; drop the leading + # speech-delay warm-up frames. + real = audio_codes[prompt_len + self.speech_delay :] + eos_rows = (real == self.audio_eos_id).any(dim=1).nonzero() + if eos_rows.numel() > 0: + real_count = int(eos_rows[0].item()) # exclude the EOS frame + eos_found = True + else: + real_count = real.shape[0] + + while real_count - sent >= threshold: + ctx = min(sent, L) + chunk = real[sent - ctx : sent + threshold] + codec_q.put((chunk, ctx, False)) + sent += threshold + threshold = self.codec_chunk_size - L + if eos_found: + break + + if state["error"] is None: + if real is not None and real_count > sent: + ctx = min(sent, L) + codec_q.put((real[sent - ctx : real_count], ctx, True)) + else: + codec_q.put(None) + + await asyncio.wrap_future(codec_future) + if state["error"] is not None: + raise state["error"] + + t_end = time.perf_counter() + ttfa_ms = ((state["t_first_audio"] or t_end) - t_start) * 1000 + logger.info( + "rid=%s ttfa=%.1fms total=%.1fms frames=%d speaker=%s text=%r", + request_id, + ttfa_ms, + (t_end - t_start) * 1000, + sent, + speaker, + text[:120], + ) + except Exception as e: + logger.error("rid=%s failed: %s", request_id, e, exc_info=True) + try: + await self.omni.abort(request_id) + except Exception: + pass + if not codec_future.done(): + codec_q.put(None) + try: + await asyncio.wrap_future(codec_future) + except Exception: + pass + self._send_error(response_sender, e) + + @staticmethod + def _read_str(request, name: str, default: str) -> str: + tensor = pb_utils.get_input_tensor_by_name(request, name) + if tensor is None: + return default + return tensor.as_numpy().flatten()[0].decode("utf-8") + + def execute(self, requests): + for request in requests: + response_sender = request.get_response_sender() + try: + text = self._read_str(request, "text", "") + context_text = self._read_str(request, "context_text", self.default_context_text) + speaker = self._read_str(request, "speaker", self.default_speaker) + asyncio.run_coroutine_threadsafe( + self._synthesize(text, context_text, speaker, response_sender), + self._loop, + ) + except Exception as e: + logger.error("Request parse failed: %s", e, exc_info=True) + self._send_error(response_sender, e) + return None + + def finalize(self): + if hasattr(self, "omni"): + try: + self.omni.shutdown() + except Exception: + pass + if hasattr(self, "_loop") and self._loop.is_running(): + self._loop.call_soon_threadsafe(self._loop.stop) + if hasattr(self, "_loop_thread"): + self._loop_thread.join(timeout=10) + if hasattr(self, "_codec_pool"): + self._codec_pool.shutdown(wait=False) + if getattr(self, "_stage_cfg_path", None): + try: + os.unlink(self._stage_cfg_path) + except OSError: + pass diff --git a/examples/tts/easymagpie_vllm_omni/model_repository/easymp/config.pbtxt b/examples/tts/easymagpie_vllm_omni/model_repository/easymp/config.pbtxt new file mode 100644 index 000000000000..f4266539dc7b --- /dev/null +++ b/examples/tts/easymagpie_vllm_omni/model_repository/easymp/config.pbtxt @@ -0,0 +1,103 @@ +name: "easymp" +backend: "python" +max_batch_size: 32 + +input [ + { + name: "text" + data_type: TYPE_STRING + dims: [ 1 ] + }, + { + name: "context_text" + data_type: TYPE_STRING + dims: [ 1 ] + optional: true + }, + { + name: "speaker" + data_type: TYPE_STRING + dims: [ 1 ] + optional: true + } +] + +output [ + { + name: "audio" + data_type: TYPE_FP32 + dims: [ -1 ] + } +] + +model_transaction_policy { + decoupled: true +} + +instance_group [ + { + count: 1 + kind: KIND_GPU + } +] + +# Converted EasyMagpie checkpoint dir (config.json + weights + tokenizer + +# speaker_embeddings/.pt), produced by easy_magpietts_convert_to_vllm.py. +parameters { + key: "vllm_model_path" + value: { string_value: "/workspace/examples/tts/easymagpie_vllm_omni/easymp_vllm_model" } +} +parameters { + key: "default_speaker" + value: { string_value: "eng" } +} +parameters { + key: "default_context_text" + value: { string_value: "[EN]" } +} + +parameters { + key: "max_model_len" + value: { string_value: "1024" } +} +parameters { + key: "max_num_seqs" + value: { string_value: "32" } +} +parameters { + key: "max_num_batched_tokens" + value: { string_value: "1024" } +} +parameters { + key: "max_new_tokens" + value: { string_value: "512" } +} +parameters { + key: "gpu_memory_utilization" + value: { string_value: "0.5" } +} + +# Codec streaming (overlap-save) in MODEL frames. codec_chunk_size must equal the +# codec engine's static frame axis (15). new frames / chunk = chunk - left_context. +parameters { + key: "codec_chunk_size" + value: { string_value: "15" } +} +parameters { + key: "codec_left_context" + value: { string_value: "12" } +} +parameters { + key: "first_chunk_frames" + value: { string_value: "2" } +} + +# Audio (local-transformer) sampling, forwarded via additional_information. +parameters { + key: "lt_temperature" + value: { string_value: "0.7" } +} +parameters { + key: "lt_top_k" + value: { string_value: "80" } +} From d11beceaf942ffaecdc75485f8ded737d3b05cb6 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimkov Date: Fri, 5 Jun 2026 12:12:53 +0200 Subject: [PATCH 16/45] examples/tts/easymagpie_vllm_omni/model_repository/easymp/1/model.py: fix sending back chunks of audio Signed-off-by: Viacheslav Klimkov --- .../model_repository/easymp/1/model.py | 87 ++++++++++--------- 1 file changed, 48 insertions(+), 39 deletions(-) diff --git a/examples/tts/easymagpie_vllm_omni/model_repository/easymp/1/model.py b/examples/tts/easymagpie_vllm_omni/model_repository/easymp/1/model.py index 237a7125e5ee..6253b4ec23c9 100644 --- a/examples/tts/easymagpie_vllm_omni/model_repository/easymp/1/model.py +++ b/examples/tts/easymagpie_vllm_omni/model_repository/easymp/1/model.py @@ -295,59 +295,68 @@ async def _synthesize(self, text: str, context_text: str, speaker: str, response t_start = time.perf_counter() request_id = f"easymp-{uuid.uuid4().hex[:8]}" prompt = self._build_prompt(text, context_text, speaker) - prompt_len = len(prompt["prompt_token_ids"]) codec_q: queue.Queue = queue.Queue() state: dict = {"t_first_audio": None, "error": None} codec_future = self._codec_pool.submit(self._codec_worker, codec_q, response_sender, state) + # The omni accumulator yields a tensor for the prefill (first) and the + # consolidated (final) steps, but a growing list during decode (one + # (1, C*S) row appended per AR step). We only consume the list yields; + # ``mm_codes[0]`` is the prefill prefix, ``mm_codes[1 + d]`` is decode + # frame d. Real audio starts after ``speech_delay`` warm-up frames. L = self.codec_left_context - sent = 0 # real frames (post speech-delay, pre EOS) already queued + base = 1 + self.speech_delay # mm_codes index of the first real frame + sent = 0 # real frames already queued to the codec threshold = self.first_chunk_frames - real: torch.Tensor | None = None - real_count = 0 - eos_found = False + mm_codes: list | None = None + produced_final = False + + def emit_ready(codes_list: list, real_count: int, final: bool) -> None: + """Queue overlap-save windows of newly-ready real frames.""" + nonlocal sent, threshold, produced_final + while sent < real_count: + remaining = real_count - sent + if not final and remaining < threshold: + break + take = min(threshold, remaining) + ctx = min(sent, L) + chunk = torch.cat(codes_list[base + sent - ctx : base + sent + take], dim=0) + sent += take + threshold = self.codec_chunk_size - L + is_final = final and sent >= real_count + codec_q.put((chunk, ctx, is_final)) + produced_final = produced_final or is_final try: async for out in self.omni.generate(prompt, request_id=request_id): if state["error"] is not None: break - mm = getattr(out, "multimodal_output", None) or {} - audio_codes = mm.get("audio_codes") - if not isinstance(audio_codes, torch.Tensor): - continue - # audio_codes accumulates one row per flat-batch token: prompt_len - # prefill rows + one per decode step. Count decoded frames from the - # tensor (token_ids on a streaming step is a delta, not cumulative). - num_decoded = audio_codes.shape[0] - prompt_len - if num_decoded <= self.speech_delay: + payload = (getattr(out, "multimodal_output", None) or {}).get("audio_codes") + if not isinstance(payload, list): continue - - # Decoded rows are everything after prefill; drop the leading - # speech-delay warm-up frames. - real = audio_codes[prompt_len + self.speech_delay :] - eos_rows = (real == self.audio_eos_id).any(dim=1).nonzero() - if eos_rows.numel() > 0: - real_count = int(eos_rows[0].item()) # exclude the EOS frame - eos_found = True - else: - real_count = real.shape[0] - - while real_count - sent >= threshold: - ctx = min(sent, L) - chunk = real[sent - ctx : sent + threshold] - codec_q.put((chunk, ctx, False)) - sent += threshold - threshold = self.codec_chunk_size - L - if eos_found: - break - - if state["error"] is None: - if real is not None and real_count > sent: - ctx = min(sent, L) - codec_q.put((real[sent - ctx : real_count], ctx, True)) - else: + mm_codes = payload + # Hold back the most recent decode frame: the audio-EOS frame is + # always the last one, and must not be vocoded. + real_avail = (len(mm_codes) - 1) - self.speech_delay - 1 + if real_avail > sent: + emit_ready(mm_codes, real_avail, final=False) + + if state["error"] is None and mm_codes is not None: + # Authoritative tail: scan for the audio-EOS frame (only it carries + # audio_eos_id > codebook_size) and vocode every real frame before it. + eos_idx = None + for i in range(len(mm_codes) - 1, 0, -1): + if bool((mm_codes[i] == self.audio_eos_id).any()): + eos_idx = i + break + last_excl = eos_idx if eos_idx is not None else len(mm_codes) + real_count = (last_excl - 1) - self.speech_delay + emit_ready(mm_codes, real_count, final=True) + if not produced_final: codec_q.put(None) + elif state["error"] is None: + codec_q.put(None) await asyncio.wrap_future(codec_future) if state["error"] is not None: From 2e102482e2027e4e1d4b2b778e9239975c9efee8 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimkov Date: Fri, 5 Jun 2026 12:13:13 +0200 Subject: [PATCH 17/45] examples/tts/easymagpie_vllm_omni/Dockerfile: add installation of the model definition Signed-off-by: Viacheslav Klimkov --- examples/tts/easymagpie_vllm_omni/Dockerfile | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/examples/tts/easymagpie_vllm_omni/Dockerfile b/examples/tts/easymagpie_vllm_omni/Dockerfile index f9b29c757618..cabf76ac4266 100644 --- a/examples/tts/easymagpie_vllm_omni/Dockerfile +++ b/examples/tts/easymagpie_vllm_omni/Dockerfile @@ -9,7 +9,13 @@ RUN pip install --no-cache-dir \ "vllm==0.21.0" \ "vllm_omni==0.21.0rc1" -# 3. TODO install NeMo/examples/tts/easymagpie_vllm_omni +# 3. Install easymagpie vllm omni model definition +# TODO: replace this clone with the upstream NeMo repo once the code is merged +RUN git clone --depth 1 --branch easymp_vllm_omni \ + https://github.com/vklimkov-nvidia/NeMoDuplexRealtime.git \ + /tmp/NeMoDuplexRealtime && \ + pip install --no-cache-dir /tmp/NeMoDuplexRealtime/examples/tts/easymagpie_vllm_omni && \ + rm -rf /tmp/NeMoDuplexRealtime # 4. Extra python requirements needed to compile the model RUN pip install --no-cache-dir \ From d5d8dd611c8cb3a52d55873dfc5e75395a5bf4a1 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimkov Date: Fri, 5 Jun 2026 12:48:22 +0200 Subject: [PATCH 18/45] examples/tts/easymagpie_vllm_omni/README.md: add info on service Signed-off-by: Viacheslav Klimkov --- examples/tts/easymagpie_vllm_omni/README.md | 106 +++++++++++++++----- 1 file changed, 79 insertions(+), 27 deletions(-) diff --git a/examples/tts/easymagpie_vllm_omni/README.md b/examples/tts/easymagpie_vllm_omni/README.md index 27cc34a08d34..8d967f1aa5c1 100644 --- a/examples/tts/easymagpie_vllm_omni/README.md +++ b/examples/tts/easymagpie_vllm_omni/README.md @@ -1,27 +1,79 @@ -WIP model definition of EasyMP for vllm-omni. Follows footsteps of qwen3tts: -backbone and LT are compiled into a single cuda graph during uniform batch decoding, -piecewise during mixed/prefill. - -Install: -``` -pip install -e ".[all]" -pip install ninja mamba_ssm causal_conv1d --no-build-isolation -# install vllm -pip install vllm==0.21.0 vllm_omni==0.21.0rc1 -# register vllm models -pip install -e examples/tts/easymagpie_vllm_omni/ -``` - -Conver the checkpoint from -https://huggingface.co/nvidia/easymagpietts_NEXT/tree/main/2605_NemotronTTS_V0.2/v2 -``` -python examples/tts/easymagpie_vllm_omni/easy_magpietts_convert_to_vllm.py \ - --nemo_file /2605_EMTTS_SmallMamba_Step150k_posttrained_epoch12.nemo \ - --codec_model_path /25fps_spectral_codec_with_bandwidth_extension.nemo \ - --outdir examples/tts/easymagpie_vllm_omni/easymp_vllm_model \ - --context_audio english_sample.wav --speaker_name eng \ - --phoneme_tokenizer_path /bpe_ipa_tokenizer_2048_en_de_es_fr_hi_it_vi_zh_ko-KR_pt-BR_ar.json -``` - -Finally run notebook `examples/tts/easymagpie_vllm_omni/easymagpie_inference_demo.ipynb` -to predict acoustic tokens \ No newline at end of file +## EasyMagpie TTS — vLLM-Omni + Triton service + +Streaming TTS server for **EasyMagpieTTS** (NeMo model +`nemo.collections.tts.models.easy_magpietts.EasyMagpieTTSModel` / +`EasyMagpieTTSInferenceModel`, Nemotron-H backbone + per-codebook local +transformer over a 25 fps spectral codec). + +The vLLM-Omni model definition (talker that runs the backbone + local +transformer as a single CUDA graph during uniform-batch decoding, piecewise +during prefill/mixed) lives in +[`vllm_plugin_easymagpie_omni/`](vllm_plugin_easymagpie_omni). A Triton +ensemble wraps it together with a TensorRT codec decoder to serve gRPC +streaming requests. + +### Pipeline + +1. **Convert the NeMo checkpoint to a vLLM-Omni model directory** — bakes the + text embedding + CAS lookup, dumps `config.json`, `model.safetensors`, the + text tokenizer, and optional reference speaker embeddings. + + ```bash + python examples/tts/easymagpie_vllm_omni/easy_magpietts_convert_to_vllm.py \ + --nemo_file /2605_EMTTS_SmallMamba_Step150k_posttrained_epoch12.nemo \ + --codec_model_path /25fps_spectral_codec_with_bandwidth_extension.nemo \ + --outdir examples/tts/easymagpie_vllm_omni/easymp_vllm_model \ + --context_audio english_sample.wav --speaker_name eng \ + --phoneme_tokenizer_path /bpe_ipa_tokenizer_2048_en_de_es_fr_hi_it_vi_zh_ko-KR_pt-BR_ar.json + ``` + + Checkpoints: . + +2. **Export the codec decoder to ONNX** — wraps `AudioCodecModel` so a single + `(B, T, C*S)` int tensor of stacked model codes decodes to a 22.05 kHz + waveform (clamp specials → unstack → FSQ index-convert → decode baked in). + + ```bash + python examples/tts/easymagpie_vllm_omni/export_codec_decoder_onnx.py \ + --codec_model_path /25fps_spectral_codec_with_bandwidth_extension.nemo \ + --nemo_file /2605_EMTTS_SmallMamba_Step150k_posttrained_epoch12.nemo \ + --onnx-path examples/tts/easymagpie_vllm_omni/codec.onnx \ + --frames 15 --device cuda + ``` + +3. **Build the serving container** (Triton 26.02 + vLLM 0.21.0 + + vllm-omni 0.21.0rc1 + this plugin). + + ```bash + docker build -t easymp-vllm-omni examples/tts/easymagpie_vllm_omni/ + ``` + +4. **Launch the container** with the workspace and a GPU mounted. + + ```bash + docker run --rm -it --gpus all --network host --shm-size=8g \ + -v "$PWD":/workspace -w /workspace \ + easymp-vllm-omni bash + ``` + +5. **Build the TensorRT engine from the ONNX** (inside the container) and drop + it into the Triton repo as `model.plan`. + + ```bash + python examples/tts/easymagpie_vllm_omni/export_codec_decoder_trt.py \ + --onnx-path examples/tts/easymagpie_vllm_omni/codec.onnx \ + --trt-path examples/tts/easymagpie_vllm_omni/model_repository/codec/1/model.plan \ + --batch-profile 1 8 32 --frames-profile 15 15 15 + ``` + +6. **Start the Triton inference server** against + [`model_repository/`](model_repository) (two models: `easymp` python + backend + `codec` TRT plan). + + ```bash + tritonserver --model-repository=examples/tts/easymagpie_vllm_omni/model_repository + ``` + +7. **Send a request.** End-to-end gRPC streaming example in + [`run_server_request.ipynb`](run_server_request.ipynb) — sends `text`, + receives streamed `audio` chunks at 22.05 kHz. From 5dd4ad2bf4159f5e218c9e627979a6ff04d85d6d Mon Sep 17 00:00:00 2001 From: Viacheslav Klimkov Date: Fri, 5 Jun 2026 13:14:18 +0200 Subject: [PATCH 19/45] examples/tts/easymagpie_vllm_omni: add benchmarks Signed-off-by: Viacheslav Klimkov --- ...k_easymagpie_tts.py => benchmark_model.py} | 0 .../easymagpie_vllm_omni/benchmark_service.py | 317 ++++++++++++++++++ 2 files changed, 317 insertions(+) rename examples/tts/easymagpie_vllm_omni/{benchmark_easymagpie_tts.py => benchmark_model.py} (100%) create mode 100644 examples/tts/easymagpie_vllm_omni/benchmark_service.py diff --git a/examples/tts/easymagpie_vllm_omni/benchmark_easymagpie_tts.py b/examples/tts/easymagpie_vllm_omni/benchmark_model.py similarity index 100% rename from examples/tts/easymagpie_vllm_omni/benchmark_easymagpie_tts.py rename to examples/tts/easymagpie_vllm_omni/benchmark_model.py diff --git a/examples/tts/easymagpie_vllm_omni/benchmark_service.py b/examples/tts/easymagpie_vllm_omni/benchmark_service.py new file mode 100644 index 000000000000..ac002b853495 --- /dev/null +++ b/examples/tts/easymagpie_vllm_omni/benchmark_service.py @@ -0,0 +1,317 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Benchmark script for the EasyMagpie TTS Triton server (decoupled mode, gRPC). + +Spawns N concurrent workers that send TTS requests in parallel against the +``easymp`` Triton model (see ``model_repository/easymp/config.pbtxt``). +Each line of the text file is parsed as ``\\t``. +Texts are randomly sampled for each request. + +Usage: + python benchmark_easymagpie_triton.py --text-file vctk_subset.txt --num-requests 100 --num-workers 8 + python benchmark_easymagpie_triton.py --text-file vctk_subset.txt --num-requests 50 \ + --output-dir out_wavs +""" + +import argparse +import queue +import random +import threading +import time +import wave +from dataclasses import dataclass, field +from pathlib import Path + +import numpy as np +import tritonclient.grpc as grpcclient + +SAMPLE_RATE = 22_050 # codec output_sample_rate (matches run_server_request.ipynb) +MODEL_NAME = "easymp" + + +@dataclass +class RequestResult: + uttid: str + num_samples: int + duration_s: float + ttfa_s: float = 0.0 + error: str | None = None + + +@dataclass +class BenchmarkStats: + lock: threading.Lock = field(default_factory=threading.Lock) + results: list[RequestResult] = field(default_factory=list) + + def add(self, result: RequestResult): + with self.lock: + self.results.append(result) + + +def _save_wav(path: Path, audio: np.ndarray, sample_rate: int = SAMPLE_RATE): + audio = np.clip(audio, -1.0, 1.0) + pcm = (audio * 32767.0).astype(np.int16) + with wave.open(str(path), "wb") as wf: + wf.setnchannels(1) + wf.setsampwidth(2) + wf.setframerate(sample_rate) + wf.writeframes(pcm.tobytes()) + + +def synthesize( + client: grpcclient.InferenceServerClient, + result_q: queue.Queue, + text: str, + chunk_timeout: float, +): + """Send one TTS request and collect streamed chunks. + + Returns ``(audio, ttfa_s, elapsed_s, error)``. + """ + text_input = grpcclient.InferInput("text", [1, 1], "BYTES") + text_input.set_data_from_numpy(np.array([[text]], dtype=object)) + + t0 = time.perf_counter() + t_first: float | None = None + chunks: list[np.ndarray] = [] + + client.async_stream_infer( + model_name=MODEL_NAME, + inputs=[text_input], + outputs=[grpcclient.InferRequestedOutput("audio")], + ) + + while True: + try: + result, error = result_q.get(timeout=chunk_timeout) + except queue.Empty: + elapsed = time.perf_counter() - t0 + return None, elapsed, elapsed, "no chunk within chunk_timeout" + + if error: + elapsed = time.perf_counter() - t0 + return None, elapsed, elapsed, str(error) + + audio = result.as_numpy("audio").squeeze() + if audio.size > 0: + if t_first is None: + t_first = time.perf_counter() + chunks.append(audio) + + response = result.get_response() + final_param = response.parameters.get("triton_final_response") + if final_param and getattr(final_param, "bool_param", False): + break + + elapsed = time.perf_counter() - t0 + ttfa = (t_first - t0) if t_first else elapsed + audio = np.concatenate(chunks) if chunks else np.array([], dtype=np.float32) + return audio, ttfa, elapsed, None + + +def worker( + worker_id: int, + triton_url: str, + items: list[tuple[str, str]], + task_queue: list[int], + queue_lock: threading.Lock, + stats: BenchmarkStats, + chunk_timeout: float, + output_dir: Path | None, +): + result_q: queue.Queue = queue.Queue() + client = grpcclient.InferenceServerClient(url=triton_url) + client.start_stream(callback=lambda result, error: result_q.put((result, error))) + + try: + while True: + with queue_lock: + if not task_queue: + return + task_idx = task_queue.pop() + + uttid, text = random.choice(items) + audio, ttfa, elapsed, error = synthesize(client, result_q, text, chunk_timeout) + + if error is not None: + # Reset the stream so late chunks don't bleed into the next + # request. + client.stop_stream() + client.start_stream(callback=lambda result, error: result_q.put((result, error))) + stats.add( + RequestResult( + uttid=uttid, + num_samples=0, + duration_s=elapsed, + ttfa_s=ttfa, + error=error, + ) + ) + print(f"[worker {worker_id:02d}] request {task_idx} ({uttid}) FAILED ({elapsed:.1f}s) — {error}") + continue + + num_samples = len(audio) + if output_dir is not None and num_samples > 0: + _save_wav(output_dir / f"{uttid}.wav", audio) + + stats.add( + RequestResult( + uttid=uttid, + num_samples=num_samples, + duration_s=elapsed, + ttfa_s=ttfa, + ) + ) + print( + f"[worker {worker_id:02d}] request {task_idx} ({uttid}) done — " + f"{num_samples / SAMPLE_RATE:.2f}s audio in {elapsed:.2f}s " + f"(TTFA: {ttfa:.3f}s)" + ) + finally: + client.stop_stream() + + +def _load_items(text_file: str) -> list[tuple[str, str]]: + items: list[tuple[str, str]] = [] + with open(text_file) as f: + for line in f: + line = line.rstrip("\n") + if not line.strip(): + continue + parts = line.split("\t", 1) + if len(parts) != 2: + raise ValueError(f"Expected '\\t' per line, got: {line!r}") + uttid, text = parts[0].strip(), parts[1].strip() + if not uttid or not text: + raise ValueError(f"Empty uttid or text in line: {line!r}") + items.append((uttid, text)) + return items + + +def _run_workers( + num_workers: int, + triton_url: str, + items: list[tuple[str, str]], + num_tasks: int, + chunk_timeout: float, + output_dir: Path | None, +) -> tuple[BenchmarkStats, float]: + task_queue = list(range(num_tasks)) + queue_lock = threading.Lock() + stats = BenchmarkStats() + + threads = [ + threading.Thread( + target=worker, + args=(i, triton_url, items, task_queue, queue_lock, stats, chunk_timeout, output_dir), + ) + for i in range(num_workers) + ] + wall_start = time.perf_counter() + for t in threads: + t.start() + for t in threads: + t.join() + return stats, time.perf_counter() - wall_start + + +def main(): + parser = argparse.ArgumentParser(description="Benchmark EasyMagpie TTS Triton server") + parser.add_argument("--text-file", required=True, help="Path to file with '\\t' per line") + parser.add_argument("--num-requests", type=int, required=True, help="Total number of requests to send") + parser.add_argument("--num-workers", type=int, default=4, help="Number of concurrent workers (default: 4)") + parser.add_argument( + "--triton-url", default="localhost:8001", help="Triton gRPC endpoint (default: localhost:8001)" + ) + parser.add_argument("--no-warmup", action="store_true", help="Skip warmup phase (3 requests per worker)") + parser.add_argument( + "--chunk-timeout", type=float, default=60, help="Per-chunk receive timeout in seconds (default: 60)" + ) + parser.add_argument( + "--output-dir", default=None, help="If set, write each generated waveform to /.wav" + ) + args = parser.parse_args() + + items = _load_items(args.text_file) + if not items: + print(f"ERROR: no usable lines found in {args.text_file}") + return + + output_dir: Path | None = None + if args.output_dir is not None: + output_dir = Path(args.output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + + print(f"Loaded {len(items)} utterances from {args.text_file}") + print(f"Sending {args.num_requests} requests with {args.num_workers} workers to {args.triton_url}") + if output_dir is not None: + print(f"Writing WAVs to {output_dir.resolve()}") + print("-" * 70) + + if not args.no_warmup: + total_warmup = args.num_workers * 3 + print(f"Warmup: {total_warmup} requests (3 per worker) ...") + _run_workers( + args.num_workers, + args.triton_url, + items, + total_warmup, + args.chunk_timeout, + output_dir=None, + ) + print("Warmup complete.") + print("-" * 70) + + stats, wall_elapsed = _run_workers( + args.num_workers, + args.triton_url, + items, + args.num_requests, + args.chunk_timeout, + output_dir, + ) + + successes = [r for r in stats.results if r.error is None] + failures = [r for r in stats.results if r.error is not None] + total_audio_seconds = sum(r.num_samples for r in successes) / SAMPLE_RATE + + print() + print("=" * 70) + print("BENCHMARK RESULTS") + print("=" * 70) + print(f" Total requests sent: {args.num_requests}") + print(f" Successful: {len(successes)}") + print(f" Failed: {len(failures)}") + print(f" Concurrent workers: {args.num_workers}") + print() + print(f" Wall-clock time: {wall_elapsed:.2f} s") + print(f" Total audio synthesized: {total_audio_seconds:.2f} s") + print(f" Real-time factor (RTF): {total_audio_seconds / wall_elapsed:.2f}x") + print(f" Throughput: {len(successes) / wall_elapsed:.2f} requests/s") + + if successes: + ttfas_ms = sorted(r.ttfa_s * 1000 for r in successes) + mean_ttfa = sum(ttfas_ms) / len(ttfas_ms) + print() + print(" Time to first audio (TTFA):") + print(f" mean: {mean_ttfa:.1f} ms") + print(f" p95: {ttfas_ms[int(len(ttfas_ms) * 0.95)]:.1f} ms") + + print("=" * 70) + + +if __name__ == "__main__": + main() From ec500735c87a0db17e41c264a09be316a5fed1c3 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimkov Date: Fri, 5 Jun 2026 13:36:59 +0200 Subject: [PATCH 20/45] examples/tts/easymagpie_vllm_omni/benchmark_model.py: reduce default memory utilization Signed-off-by: Viacheslav Klimkov --- examples/tts/easymagpie_vllm_omni/benchmark_model.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/tts/easymagpie_vllm_omni/benchmark_model.py b/examples/tts/easymagpie_vllm_omni/benchmark_model.py index 7abe8b296143..53755c1dd488 100644 --- a/examples/tts/easymagpie_vllm_omni/benchmark_model.py +++ b/examples/tts/easymagpie_vllm_omni/benchmark_model.py @@ -116,7 +116,7 @@ def _build_easymagpie_stage_config( torch_profiler_dir: str = "./profiler_traces", with_stack: bool = False, record_shapes: bool = False, - gpu_memory_utilization: float = 0.8, + gpu_memory_utilization: float = 0.5, max_model_len: int = 1024, max_num_batched_tokens: int = 1024, enforce_eager: bool = False, @@ -1042,7 +1042,7 @@ def parse_args(): ) engine = parser.add_argument_group("engine") - engine.add_argument("--gpu-memory-utilization", type=float, default=0.8) + engine.add_argument("--gpu-memory-utilization", type=float, default=0.5) engine.add_argument("--max-model-len", type=int, default=1024) engine.add_argument("--max-num-batched-tokens", type=int, default=1024) engine.add_argument("--dtype", type=str, default="float16", help="Model dtype (float16 / bfloat16)") From 235c9eed0d39d404ac835bacf24da50a0174e077 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimkov Date: Fri, 5 Jun 2026 13:37:28 +0200 Subject: [PATCH 21/45] examples/tts/easymagpie_vllm_omni/Dockerfile: try to fix warning about other plugin Signed-off-by: Viacheslav Klimkov --- examples/tts/easymagpie_vllm_omni/Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/examples/tts/easymagpie_vllm_omni/Dockerfile b/examples/tts/easymagpie_vllm_omni/Dockerfile index cabf76ac4266..4fc4ef571be0 100644 --- a/examples/tts/easymagpie_vllm_omni/Dockerfile +++ b/examples/tts/easymagpie_vllm_omni/Dockerfile @@ -26,4 +26,7 @@ RUN pip install --no-cache-dir \ "tritonclient[grpc]" RUN pip install --no-cache-dir --force-reinstall --no-deps "numpy==2.3.5" +# 5. Restrict vLLM plugin auto-discovery to the EasyMagpie plugin only. +ENV VLLM_PLUGINS=easymagpie_omni + WORKDIR /workspace From 8d3c65ad6485f8f1f7824619cf94d9b05b3c7d16 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimkov Date: Fri, 5 Jun 2026 14:12:47 +0200 Subject: [PATCH 22/45] examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/easymagpie.py: fix preprocessing start_idx usage Signed-off-by: Viacheslav Klimkov --- .../easymagpie_vllm_omni/easymagpie.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/easymagpie.py b/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/easymagpie.py index 5c6c29a17d0f..0b12af45faca 100644 --- a/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/easymagpie.py +++ b/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/easymagpie.py @@ -612,8 +612,19 @@ def preprocess( if span_len > 1: return self._preprocess_prefill(input_ids, span_len, device, info_dict) + + start = self._batch_slot_offset(input_ids, start) return self._preprocess_decode(input_ids, start, device, info_dict) + @staticmethod + def _batch_slot_offset(input_ids_view: torch.Tensor, fallback: int) -> int: + """Recover a request's batch-row offset from its 1-D ``input_ids`` view. + The runner passes ``input_ids = input_ids_buffer[s:e]`` + """ + if input_ids_view.dim() == 1 and input_ids_view.is_contiguous(): + return int(input_ids_view.storage_offset()) + return int(fallback) + def _preprocess_prefill( self, input_ids: torch.Tensor, From dbe1374e5718983ae82e70774eac577084fc7fe0 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimkov Date: Fri, 5 Jun 2026 14:21:39 +0200 Subject: [PATCH 23/45] examples/tts/easymagpie_vllm_omni/model_repository/easymp/config.pbtxt: increased batched tokens in case a lot of simaltenious prefill Signed-off-by: Viacheslav Klimkov --- .../easymagpie_vllm_omni/model_repository/easymp/config.pbtxt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/tts/easymagpie_vllm_omni/model_repository/easymp/config.pbtxt b/examples/tts/easymagpie_vllm_omni/model_repository/easymp/config.pbtxt index f4266539dc7b..bc5f70d3bcf0 100644 --- a/examples/tts/easymagpie_vllm_omni/model_repository/easymp/config.pbtxt +++ b/examples/tts/easymagpie_vllm_omni/model_repository/easymp/config.pbtxt @@ -66,7 +66,7 @@ parameters { } parameters { key: "max_num_batched_tokens" - value: { string_value: "1024" } + value: { string_value: "4096" } } parameters { key: "max_new_tokens" From 0f90c2aefb23a5ca7480d61fe14219bccf3a74d9 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimkov Date: Fri, 5 Jun 2026 14:27:08 +0200 Subject: [PATCH 24/45] examples/tts/easymagpie_vllm_omni: small tweaks Signed-off-by: Viacheslav Klimkov --- examples/tts/easymagpie_vllm_omni/README.md | 6 +- .../run_service_request.ipynb | 133 ++++++++++++++++++ 2 files changed, 136 insertions(+), 3 deletions(-) create mode 100644 examples/tts/easymagpie_vllm_omni/run_service_request.ipynb diff --git a/examples/tts/easymagpie_vllm_omni/README.md b/examples/tts/easymagpie_vllm_omni/README.md index 8d967f1aa5c1..df7970a8687d 100644 --- a/examples/tts/easymagpie_vllm_omni/README.md +++ b/examples/tts/easymagpie_vllm_omni/README.md @@ -57,13 +57,13 @@ streaming requests. ``` 5. **Build the TensorRT engine from the ONNX** (inside the container) and drop - it into the Triton repo as `model.plan`. + it into the Triton repo as `model.plan`. For now fp32 seems to be mandatory. ```bash python examples/tts/easymagpie_vllm_omni/export_codec_decoder_trt.py \ --onnx-path examples/tts/easymagpie_vllm_omni/codec.onnx \ --trt-path examples/tts/easymagpie_vllm_omni/model_repository/codec/1/model.plan \ - --batch-profile 1 8 32 --frames-profile 15 15 15 + --batch-profile 1 8 32 --frames-profile 15 15 15 --fp32 ``` 6. **Start the Triton inference server** against @@ -75,5 +75,5 @@ streaming requests. ``` 7. **Send a request.** End-to-end gRPC streaming example in - [`run_server_request.ipynb`](run_server_request.ipynb) — sends `text`, + [`run_server_request.ipynb`](run_service_request.ipynb) — sends `text`, receives streamed `audio` chunks at 22.05 kHz. diff --git a/examples/tts/easymagpie_vllm_omni/run_service_request.ipynb b/examples/tts/easymagpie_vllm_omni/run_service_request.ipynb new file mode 100644 index 000000000000..3517294521fe --- /dev/null +++ b/examples/tts/easymagpie_vllm_omni/run_service_request.ipynb @@ -0,0 +1,133 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "121f7df0", + "metadata": {}, + "outputs": [], + "source": [ + "import queue\n", + "import time\n", + "\n", + "import numpy as np\n", + "import tritonclient.grpc as grpcclient\n", + "import matplotlib.pyplot as plt\n", + "from IPython.display import Audio, display\n", + "\n", + "TRITON_URL = \"localhost:8001\"\n", + "MODEL_NAME = \"easymp\"\n", + "SAMPLE_RATE = 22050 # codec output_sample_rate" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5917fcfa", + "metadata": {}, + "outputs": [], + "source": [ + "def synthesize(text: str) -> np.ndarray:\n", + " \"\"\"Send a TTS request and collect streamed audio chunks.\"\"\"\n", + " result_q: queue.Queue = queue.Queue()\n", + "\n", + " def _on_response(result, error):\n", + " result_q.put((result, error))\n", + "\n", + " client = grpcclient.InferenceServerClient(url=TRITON_URL)\n", + " client.start_stream(callback=_on_response)\n", + "\n", + " text_input = grpcclient.InferInput(\"text\", [1, 1], \"BYTES\")\n", + " text_input.set_data_from_numpy(np.array([[text]], dtype=object))\n", + "\n", + " client.async_stream_infer(\n", + " model_name=MODEL_NAME,\n", + " inputs=[text_input],\n", + " outputs=[grpcclient.InferRequestedOutput(\"audio\")],\n", + " )\n", + "\n", + " chunks = []\n", + " t0 = time.perf_counter()\n", + " t_first = None\n", + "\n", + " while True:\n", + " result, error = result_q.get(timeout=120)\n", + " if error:\n", + " client.stop_stream()\n", + " raise RuntimeError(str(error))\n", + "\n", + " audio = result.as_numpy(\"audio\").squeeze()\n", + " if audio.size > 0:\n", + " if t_first is None:\n", + " t_first = time.perf_counter()\n", + " chunks.append(audio)\n", + "\n", + " response = result.get_response()\n", + " final_param = response.parameters.get(\"triton_final_response\")\n", + " if final_param and getattr(final_param, \"bool_param\", False):\n", + " break\n", + "\n", + " client.stop_stream()\n", + "\n", + " elapsed = time.perf_counter() - t0\n", + " ttfa = (t_first - t0) if t_first else elapsed\n", + " print(f\"Received {len(chunks)} chunks | TTFA: {ttfa:.3f}s | total: {elapsed:.3f}s\")\n", + " return np.concatenate(chunks) if chunks else np.array([], dtype=np.float32)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4ffa9f39", + "metadata": {}, + "outputs": [], + "source": [ + "audio = synthesize(\n", + " text=\"Since then physicists have found that it is not reflection, but refraction by the raindrops which causes the rainbows.\",\n", + ")\n", + "print(f\"Got {len(audio)} samples — {len(audio) / SAMPLE_RATE:.2f}s @ {SAMPLE_RATE} Hz\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "49843485", + "metadata": {}, + "outputs": [], + "source": [ + "t = np.arange(len(audio)) / SAMPLE_RATE\n", + "\n", + "fig, ax = plt.subplots(figsize=(12, 3))\n", + "ax.plot(t, audio, linewidth=0.3)\n", + "ax.set_xlabel(\"Time (s)\")\n", + "ax.set_ylabel(\"Amplitude\")\n", + "ax.set_title(\"EasyMP Waveform\")\n", + "fig.tight_layout()\n", + "plt.show()\n", + "\n", + "display(Audio(audio, rate=SAMPLE_RATE))" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "streaming_vllm", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From a1bcbe7eb65940db576771bbf0b43981fc725c66 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimkov Date: Mon, 8 Jun 2026 16:01:24 +0200 Subject: [PATCH 25/45] examples/tts/easymagpie_vllm_omni: log fails, adjust numpy installation Signed-off-by: Viacheslav Klimkov --- examples/tts/easymagpie_vllm_omni/Dockerfile | 3 ++- .../model_repository/easymp/1/model.py | 14 ++++++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/examples/tts/easymagpie_vllm_omni/Dockerfile b/examples/tts/easymagpie_vllm_omni/Dockerfile index 4fc4ef571be0..e1047c2ec77c 100644 --- a/examples/tts/easymagpie_vllm_omni/Dockerfile +++ b/examples/tts/easymagpie_vllm_omni/Dockerfile @@ -24,7 +24,8 @@ RUN pip install --no-cache-dir \ sox \ onnx-graphsurgeon \ "tritonclient[grpc]" -RUN pip install --no-cache-dir --force-reinstall --no-deps "numpy==2.3.5" +# have to force numpy after previous installations otherwise vllm_omni doesnt work +RUN pip install --no-cache-dir "numpy==2.3.5" # 5. Restrict vLLM plugin auto-discovery to the EasyMagpie plugin only. ENV VLLM_PLUGINS=easymagpie_omni diff --git a/examples/tts/easymagpie_vllm_omni/model_repository/easymp/1/model.py b/examples/tts/easymagpie_vllm_omni/model_repository/easymp/1/model.py index 6253b4ec23c9..a5f04ead6b7d 100644 --- a/examples/tts/easymagpie_vllm_omni/model_repository/easymp/1/model.py +++ b/examples/tts/easymagpie_vllm_omni/model_repository/easymp/1/model.py @@ -294,7 +294,6 @@ def _codec_worker(self, codec_q: queue.Queue, response_sender, state: dict) -> N async def _synthesize(self, text: str, context_text: str, speaker: str, response_sender): t_start = time.perf_counter() request_id = f"easymp-{uuid.uuid4().hex[:8]}" - prompt = self._build_prompt(text, context_text, speaker) codec_q: queue.Queue = queue.Queue() state: dict = {"t_first_audio": None, "error": None} @@ -329,6 +328,7 @@ def emit_ready(codes_list: list, real_count: int, final: bool) -> None: produced_final = produced_final or is_final try: + prompt = self._build_prompt(text, context_text, speaker) async for out in self.omni.generate(prompt, request_id=request_id): if state["error"] is not None: break @@ -387,6 +387,15 @@ def emit_ready(codes_list: list, real_count: int, final: bool) -> None: pass self._send_error(response_sender, e) + @staticmethod + def _log_future_exception(future: concurrent.futures.Future) -> None: + try: + exc = future.exception() + except concurrent.futures.CancelledError: + return + if exc is not None: + logger.error("_synthesize task crashed: %s", exc, exc_info=exc) + @staticmethod def _read_str(request, name: str, default: str) -> str: tensor = pb_utils.get_input_tensor_by_name(request, name) @@ -401,10 +410,11 @@ def execute(self, requests): text = self._read_str(request, "text", "") context_text = self._read_str(request, "context_text", self.default_context_text) speaker = self._read_str(request, "speaker", self.default_speaker) - asyncio.run_coroutine_threadsafe( + future = asyncio.run_coroutine_threadsafe( self._synthesize(text, context_text, speaker, response_sender), self._loop, ) + future.add_done_callback(self._log_future_exception) except Exception as e: logger.error("Request parse failed: %s", e, exc_info=True) self._send_error(response_sender, e) From 4bc5218b3ec415fdbb85e37eb56072bcc15c7b07 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimkov Date: Tue, 9 Jun 2026 12:27:39 +0200 Subject: [PATCH 26/45] examples/tts/easymagpie_vllm_omni/benchmark_model.py: allow dummy model weights Signed-off-by: Viacheslav Klimkov --- .../easymagpie_vllm_omni/benchmark_model.py | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/examples/tts/easymagpie_vllm_omni/benchmark_model.py b/examples/tts/easymagpie_vllm_omni/benchmark_model.py index 53755c1dd488..05ef4d072aa9 100644 --- a/examples/tts/easymagpie_vllm_omni/benchmark_model.py +++ b/examples/tts/easymagpie_vllm_omni/benchmark_model.py @@ -124,6 +124,7 @@ def _build_easymagpie_stage_config( dtype: str = "float16", distributed_executor_backend: str = "uni", cudagraph_mode: Optional[str] = None, + load_format: Optional[str] = None, ) -> dict: """Build a single-stage YAML dict containing only the EasyMagpie talker. @@ -170,6 +171,12 @@ def _build_easymagpie_stage_config( "skip_tokenizer_init": True, } + # Weight loading strategy. ``dummy`` initializes random weights and skips the + # checkpoint entirely — pair it with a dummy config dir (e.g. a Qwen3 backbone + # profile) to benchmark a backbone architecture without a trained checkpoint. + if load_format is not None: + engine_args["load_format"] = load_format + # CUDA-graph capture strategy. ``enforce_eager`` already disables graphs, so # only set compilation_config when graphs are enabled (mirrors the sidecar # server). Passed as a plain dict so it survives YAML serialization; vLLM @@ -787,6 +794,7 @@ async def main(args): dtype=args.dtype, distributed_executor_backend=args.distributed_executor_backend, cudagraph_mode=args.cudagraph_mode, + load_format=args.load_format, ) if args.cudagraph_mode is not None and args.enforce_eager: logger.warning( @@ -919,6 +927,19 @@ async def main(args): bench.config_name = args.config_name all_bench_results.append(asdict(bench)) + # ── Top-level summary (one line per concurrency level) ───────────── + print(f"\n{'=' * 56}") + print(f"{'Summary (' + args.config_name + ')':^56}") + print(f"{'=' * 56}") + for r in all_bench_results: + print( + f"concurrency={r['concurrency']}: " + f"req/s {r['request_throughput']:.2f}, " + f"ttft {r['mean_ttft_ms']:.2f}ms, " + f"itl {r['mean_itl_ms']:.2f}ms" + ) + print(f"{'=' * 56}\n") + # ── Save results ────────────────────────────────────────────────── if args.result_dir: result_dir = Path(args.result_dir) @@ -1042,6 +1063,15 @@ def parse_args(): ) engine = parser.add_argument_group("engine") + engine.add_argument( + "--load-format", + type=str, + default=None, + choices=["auto", "dummy", "safetensors", "pt"], + help="Weight loading strategy. Use 'dummy' to initialize random weights and skip the " + "checkpoint (pair with a dummy config dir to benchmark a backbone without a trained " + "checkpoint). Default: unset (vLLM default 'auto').", + ) engine.add_argument("--gpu-memory-utilization", type=float, default=0.5) engine.add_argument("--max-model-len", type=int, default=1024) engine.add_argument("--max-num-batched-tokens", type=int, default=1024) From 63976b3a55746761a5c72db7c0712dd5a881369b Mon Sep 17 00:00:00 2001 From: Viacheslav Klimkov Date: Tue, 9 Jun 2026 14:15:25 +0200 Subject: [PATCH 27/45] examples/tts/easymagpie_vllm_omni: return DELTA from the model Signed-off-by: Viacheslav Klimkov --- .../easymagpie_vllm_omni/benchmark_model.py | 84 +++++++++---- .../easymagpie_inference_demo.ipynb | 112 +++++++++++++----- 2 files changed, 148 insertions(+), 48 deletions(-) diff --git a/examples/tts/easymagpie_vllm_omni/benchmark_model.py b/examples/tts/easymagpie_vllm_omni/benchmark_model.py index 05ef4d072aa9..da40381776ba 100644 --- a/examples/tts/easymagpie_vllm_omni/benchmark_model.py +++ b/examples/tts/easymagpie_vllm_omni/benchmark_model.py @@ -346,6 +346,7 @@ class RequestResult: text: str = "" prompt_len: int = 0 num_generated: int = 0 # decoded frames (engine tokens) up to EOS + delta_frames: int = 0 # decode frames consumed as per-step deltas (should match num_generated) audio_frames: int = 0 # codec frames of real audio (post speech-delay, pre-EOS) audio_s: float = 0.0 # synthesized audio duration in seconds steps: int = 0 @@ -407,6 +408,31 @@ def _extract_request_output(stage_output): return getattr(stage_output, "request_output", stage_output) +def _new_audio_frames(payload, prev_num_tokens: int, cur_num_tokens: int): + """Return the decode frames new since the previous step as a ``(k, C*S)`` tensor. + + Under ``RequestOutputKind.DELTA`` the engine exposes ``audio_codes`` as a + *growing list* during decode — ``[prefill_prefix, frame_0, frame_1, ...]`` — + where element 0 is the prefill prefix and element ``1 + d`` is decode frame + ``d`` (each a ``(1, C*S)`` tensor). The new frames are therefore the list + elements past the previously consumed count. The prefill (first) and final + (consolidated) steps yield a cumulative *tensor* instead, whose newest + ``n_new`` rows are the new frames. Returns ``None`` when there is nothing new. + """ + import torch + + n_new = cur_num_tokens - prev_num_tokens + if n_new <= 0: + return None + if isinstance(payload, list): + chunks = [c for c in payload[1 + prev_num_tokens : 1 + cur_num_tokens] if isinstance(c, torch.Tensor)] + chunks = [c for c in chunks if c.numel() > 0] + return torch.cat(chunks, dim=0) if chunks else None + if isinstance(payload, torch.Tensor) and payload.shape[0] >= n_new: + return payload[-n_new:] + return None + + async def run_one_request( omni, prompt: dict, @@ -419,10 +445,16 @@ async def run_one_request( """Submit one TTS request, collect per-token timing and audio length. Each engine step yields one decoded frame (one layer-0 token). We time the - first token (TTFT) and the gaps between subsequent tokens (ITL). The audio - EOS lives in codebook 0 of the accumulated ``audio_codes`` (not in the vLLM - token stream), so we watch the newest decoded frame and stop at the EOS - frame to recover the real synthesized length. + first token (TTFT) and the gaps between subsequent tokens (ITL). + + **Delta consumption.** The request runs in ``RequestOutputKind.DELTA``, so the + engine exposes ``multimodal_output["audio_codes"]`` as a *growing list* during + decode — ``[prefill_prefix, frame_0, frame_1, ...]`` — appending one + ``(1, C*S)`` frame per step instead of re-sending the whole cumulative tensor. + We consume only the frames new since the previous step (see + :func:`_new_audio_frames`). The audio EOS lives in the codebooks (not the vLLM + token stream), so we scan only those delta frames and stop at the EOS frame to + recover the real synthesized length. """ import torch @@ -431,6 +463,7 @@ async def run_one_request( t_last_token = None prev_num_tokens = 0 eos_decode_idx = None # 0-based decode-frame index where audio EOS appears + delta_frames_total = 0 # decode frames consumed as per-step deltas (sanity check) try: gen = omni.generate( @@ -459,23 +492,26 @@ async def run_one_request( result.inter_token_latencies.append(now - t_last_token) t_last_token = now - # Audio-EOS detection on the newest decoded frame. The accumulated - # audio_codes hold (T_ctx prefill + decode) rows; the last row is - # the newest decoded frame. Only meaningful past the speech delay. + # ── Delta: process only the audio frames produced *this step* ── + # DELTA mode appends one (1, C*S) frame per step to the audio_codes + # list, so we pull just the frames new since the previous step. mm = getattr(stage_output, "multimodal_output", None) or {} - audio_codes = mm.get("audio_codes") - newest_frame_idx = cur_num_tokens - 1 # 0-based decode-frame index - if ( - eos_decode_idx is None - and newest_frame_idx >= meta.speech_delay - and isinstance(audio_codes, torch.Tensor) - and audio_codes.numel() > 0 - ): - # audio EOS in ANY codebook (not just codebook 0) — mirrors the - # reference EOS check and the model's own stop signal. - if bool((audio_codes[-1] == meta.audio_eos_id).any()): - eos_decode_idx = newest_frame_idx - result.eos_reached = True + new_frames = _new_audio_frames(mm.get("audio_codes"), prev_num_tokens, cur_num_tokens) + if new_frames is not None and new_frames.numel() > 0: + delta_frames_total += int(new_frames.shape[0]) + # Audio-EOS detection scans only the delta frames. EOS in ANY + # codebook (not just codebook 0) — mirrors the reference EOS + # check and the model's own stop signal. Only meaningful past + # the speech delay. + if eos_decode_idx is None: + for j in range(new_frames.shape[0]): + frame_idx = prev_num_tokens + j # 0-based decode-frame index + if frame_idx < meta.speech_delay: + continue + if bool((new_frames[j] == meta.audio_eos_id).any()): + eos_decode_idx = frame_idx + result.eos_reached = True + break prev_num_tokens = cur_num_tokens @@ -485,6 +521,7 @@ async def run_one_request( t_end = time.perf_counter() result.e2e_s = t_end - t_start result.num_generated = prev_num_tokens + result.delta_frames = delta_frames_total result.success = True if result.ttft_s == 0.0 and result.steps > 0: @@ -643,6 +680,7 @@ def compute_and_print_metrics( "e2e_ms": r.e2e_s * 1000, "rtx": r.rtx, "num_generated": r.num_generated, + "delta_frames": r.delta_frames, "audio_frames": r.audio_frames, "audio_s": r.audio_s, "eos_reached": r.eos_reached, @@ -727,6 +765,7 @@ def compute_and_print_metrics( async def main(args): from vllm import SamplingParams + from vllm.sampling_params import RequestOutputKind from vllm_omni import AsyncOmni model_name = args.model @@ -814,6 +853,11 @@ async def main(args): # any codebook), so vLLM stops the request there instead of decoding the # full budget. stop_token_ids is honored even with ignore_eos. stop_token_ids=[meta.stop_token_id], + # DELTA: surface audio_codes as a *growing list* of per-step frame deltas + # during decode (element 0 = prefill prefix, element 1+d = decode frame d) + # instead of re-sending the whole cumulative tensor every step. This is the + # same streaming-consumption pattern the Triton deployment uses. + output_kind=RequestOutputKind.DELTA, ) try: diff --git a/examples/tts/easymagpie_vllm_omni/easymagpie_inference_demo.ipynb b/examples/tts/easymagpie_vllm_omni/easymagpie_inference_demo.ipynb index 7aeadfaf7129..2c773c51fbfe 100644 --- a/examples/tts/easymagpie_vllm_omni/easymagpie_inference_demo.ipynb +++ b/examples/tts/easymagpie_vllm_omni/easymagpie_inference_demo.ipynb @@ -66,6 +66,7 @@ "import yaml\n", "\n", "from vllm import SamplingParams\n", + "from vllm.sampling_params import RequestOutputKind\n", "from vllm_omni import AsyncOmni\n", "\n", "# Importing the model package is optional (the engine resolves the arch via the\n", @@ -353,6 +354,11 @@ " # decoding the full DECODE_STEPS budget. stop_token_ids is honored regardless\n", " # of ignore_eos.\n", " stop_token_ids=[AUDIO_STOP_TOKEN_ID],\n", + " # DELTA: expose audio_codes as a *growing list* of per-step frame deltas during\n", + " # decode (element 0 = prefill prefix, element 1+d = decode frame d) instead of\n", + " # re-sending the whole cumulative tensor each step. This is the streaming\n", + " # consumption pattern used by the Triton deployment (model_repository/easymp).\n", + " output_kind=RequestOutputKind.DELTA,\n", ")" ] }, @@ -361,15 +367,21 @@ "id": "3ef8934d", "metadata": {}, "source": [ - "## 4. Run inference and extract audio codes\n", + "## 4. Run inference and extract audio codes (delta streaming)\n", "\n", "`omni.generate(...)` is an async generator yielding one `RequestOutput` per\n", - "engine step; we keep the last one. As in the eartts demo, the accumulated\n", - "`multimodal_output[\\\"audio_codes\\\"]` holds one row per flat-batch token over the\n", - "whole run (the `T_ctx` prefill frames — codes left zero — plus one frame per\n", - "decode step), so we trim to the last `len(token_ids)` rows to recover just the\n", - "decoded frames, then trim again at the audio EOS frame (the model signals\n", - "end-of-speech in the codes, not in the vLLM token stream)." + "engine step. Because we set `output_kind=RequestOutputKind.DELTA`, the engine\n", + "exposes `multimodal_output[\\\"audio_codes\\\"]` as a **growing list** during decode —\n", + "`[prefill_prefix, frame_0, frame_1, ...]` — appending one `(1, C*S)` frame per\n", + "step (element 0 is the prefill prefix, element `1 + d` is decode frame `d`). So\n", + "each step we collect just the *new* frames since the previous step — exactly the\n", + "per-step delta a streaming client would push to the codec frame-by-frame (this is\n", + "the same pattern the Triton deployment in `model_repository/easymp` uses). The\n", + "prefill (first) and final (consolidated) steps yield a cumulative *tensor*\n", + "instead; we keep the final consolidated tensor as the authoritative full sequence\n", + "for vocoding. We then drop the leading `speech_delay` warm-up frames and trim the\n", + "trailing audio-EOS frame (the model signals end-of-speech in the codes, not in the\n", + "vLLM token stream)." ] }, { @@ -380,9 +392,23 @@ "outputs": [], "source": [ "async def run_request(prompt: dict, sampling_params):\n", + " \"\"\"Stream the request, collecting the per-step audio-code *deltas*.\n", + "\n", + " With ``RequestOutputKind.DELTA`` the engine exposes ``audio_codes`` as a\n", + " *growing list* during decode — ``[prefill_prefix, frame_0, frame_1, ...]`` —\n", + " appending one ``(1, C*S)`` frame per step (the prefill and final steps yield a\n", + " cumulative tensor instead). We collect each new frame as it arrives (exactly\n", + " what a streaming client would push to the codec frame-by-frame) and also keep\n", + " the final consolidated tensor as the authoritative full sequence for vocoding.\n", + " \"\"\"\n", " request_id = f\"easymagpie-{uuid.uuid4().hex[:8]}\"\n", - " final_ro = None\n", + " delta_frames = [] # per-step decode-frame deltas (excludes prefill prefix)\n", + " prev_frames = 0 # decode frames already collected from the growing list\n", + " final_tensor = None # last consolidated (T_ctx + N, C*S) tensor observed\n", + " saw_list = False # whether audio_codes ever appeared in the growing-list form\n", + " cur_num_tokens = 0\n", " num_steps = 0\n", + " final_ro = None\n", " async for stage_output in omni.generate(\n", " prompt,\n", " sampling_params_list=[sampling_params],\n", @@ -390,25 +416,54 @@ " ):\n", " final_ro = stage_output\n", " num_steps += 1\n", - " return final_ro, num_steps\n", - "\n", "\n", - "final_ro, num_steps = await run_request(prompt, sampling_params)\n", + " co = stage_output.outputs[0] if stage_output.outputs else None\n", + " cum_ids = getattr(co, \"cumulative_token_ids\", None) if co else None\n", + " if cum_ids is not None:\n", + " cur_num_tokens = len(cum_ids)\n", + " elif co is not None:\n", + " cur_num_tokens = len(getattr(co, \"token_ids\", []) or [])\n", + "\n", + " payload = (stage_output.multimodal_output or {}).get(\"audio_codes\")\n", + " if isinstance(payload, list):\n", + " # DELTA decode form: element 0 is the prefill prefix; decode frame d is\n", + " # at index 1 + d. Collect frames new since the previous step.\n", + " saw_list = True\n", + " avail = len(payload) - 1\n", + " if avail > prev_frames:\n", + " for t in payload[1 + prev_frames : 1 + avail]:\n", + " if isinstance(t, torch.Tensor):\n", + " delta_frames.append(t.detach().cpu().to(torch.long))\n", + " prev_frames = avail\n", + " elif isinstance(payload, torch.Tensor) and cur_num_tokens > 0:\n", + " # prefill (first) or consolidated (final) yield: the last cur_num_tokens\n", + " # rows are the decode frames (the T_ctx prefill prefix rows lead).\n", + " final_tensor = payload[-cur_num_tokens:].detach().cpu().to(torch.long)\n", + "\n", + " return final_ro, num_steps, delta_frames, final_tensor, saw_list\n", + "\n", + "\n", + "final_ro, num_steps, delta_frames, final_tensor, saw_list = await run_request(prompt, sampling_params)\n", "assert final_ro is not None, \"no output from engine\"\n", "\n", - "mm = final_ro.multimodal_output or {}\n", - "audio_codes = mm.get(\"audio_codes\")\n", - "token_ids = final_ro.outputs[0].token_ids if final_ro.outputs else []\n", + "print(f\"Engine steps yielded : {num_steps}\")\n", + "print(\n", + " f\"Per-step delta frames seen : {len(delta_frames)} \"\n", + " f\"(audio_codes exposed as a {'growing list' if saw_list else 'tensor'})\"\n", + ")\n", + "\n", + "# Prefer the authoritative consolidated tensor (it includes the trailing EOS\n", + "# frame); fall back to the streamed deltas if no consolidated tensor was observed.\n", + "if final_tensor is not None:\n", + " audio_codes = final_tensor\n", + "elif delta_frames:\n", + " audio_codes = torch.cat(delta_frames, dim=0)\n", + "else:\n", + " audio_codes = None\n", "\n", - "print(f\"Engine steps yielded : {num_steps}\")\n", - "print(f\"Layer-0 tokens (token_ids) : {len(token_ids)}\")\n", "if isinstance(audio_codes, torch.Tensor):\n", - " audio_codes = audio_codes.detach().cpu().to(torch.long)\n", - " print(f\"audio_codes shape (raw) : {tuple(audio_codes.shape)}\")\n", - " # Trim the Tref prefill frames echoed during prefill: keep only the decoded\n", - " # frames (the last len(token_ids) rows), exactly like the eartts demo.\n", - " if len(token_ids) > 0:\n", - " audio_codes = audio_codes[-len(token_ids):].contiguous()\n", + " source = \"final consolidated tensor\" if final_tensor is not None else \"streamed deltas\"\n", + " print(f\"audio_codes shape (decode) : {tuple(audio_codes.shape)} (from {source})\")\n", "\n", " # Drop the leading streaming_speech_delay warm-up frames. With the streaming\n", " # delay the audio stream only opens at decode step == speech_delay, so the\n", @@ -427,19 +482,20 @@ " co = final_ro.outputs[0] if final_ro.outputs else None\n", " finish_reason = getattr(co, \"finish_reason\", None)\n", " stop_reason = getattr(co, \"stop_reason\", None)\n", - " print(f\"finish_reason / stop_reason: {finish_reason} / {stop_reason}\")\n", + " print(f\"finish_reason / stop_reason : {finish_reason} / {stop_reason}\")\n", " if finish_reason == \"stop\" and stop_reason == AUDIO_STOP_TOKEN_ID and audio_codes.shape[0] > 0:\n", " print(f\"dropping trailing audio-EOS frame at {audio_codes.shape[0] - 1}\")\n", " audio_codes = audio_codes[:-1].contiguous()\n", " else:\n", " print(f\"no engine EOS stop (finish_reason={finish_reason}); using full decode\")\n", "\n", - " print(f\"audio_codes shape (decode) : {tuple(audio_codes.shape)}\")\n", - " print(f\"audio_codes dtype : {audio_codes.dtype}\")\n", - " print(f\"codes min / max : {int(audio_codes.min())} / {int(audio_codes.max())}\")\n", - " print(f\"first frame codes : {audio_codes[0].tolist()}\")\n", + " print(f\"audio_codes shape (final) : {tuple(audio_codes.shape)}\")\n", + " print(f\"audio_codes dtype : {audio_codes.dtype}\")\n", + " print(f\"codes min / max : {int(audio_codes.min())} / {int(audio_codes.max())}\")\n", + " print(f\"first frame codes : {audio_codes[0].tolist()}\")\n", "else:\n", - " print(f\"audio_codes : {audio_codes!r}\")" + " audio_codes = None\n", + " print(\"audio_codes : no codes collected\")" ] }, { From 7754d909208f5eb7b4d826b0f77b196a52d72820 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimkov Date: Tue, 9 Jun 2026 22:41:57 +0200 Subject: [PATCH 28/45] examples/tts/easymagpie_vllm_omni: allow streaming text tokens in the model Signed-off-by: Viacheslav Klimkov --- .../easymagpie_vllm_omni/backbone_patches.py | 58 +++++++++ .../easymagpie_vllm_omni/easymagpie.py | 112 ++++++++++++++---- 2 files changed, 149 insertions(+), 21 deletions(-) diff --git a/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/backbone_patches.py b/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/backbone_patches.py index da57ce6742cc..faceae41c24d 100644 --- a/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/backbone_patches.py +++ b/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/backbone_patches.py @@ -23,11 +23,69 @@ import torch import torch.nn as nn import torch.nn.functional as F +import vllm.v1.attention.backends.mamba_attn as _mamba_attn from vllm.logger import init_logger logger = init_logger(__name__) +def patch_mamba_streaming_decode() -> None: + """Treat 1-token streaming extends as decodes so FULL decode cudagraphs work. + + EasyMagpie's streaming-input path keeps extending each request's prompt with + every chunk, so ``num_computed_tokens < num_prompt_tokens`` (the engine's + ``is_prefilling`` flag) stays True for the whole stream. vLLM's Mamba2 + metadata builder calls + :func:`vllm.v1.attention.backends.utils.split_decodes_and_prefills` with + ``treat_short_extends_as_decodes=False``, so every single-token decode step + is classified as a *prefill* (``num_prefills>0``). + + That collides with the cudagraph dispatcher, which keys only on query length: + a uniform ``query_len==1`` batch dispatches the **FULL decode** graph + regardless of ``is_prefilling``. Two failures result: + + * the replayed decode graph runs the decode Mamba kernels while the metadata + says prefill, and + * because ``num_prefills>0``, ``_update_metadata_for_cudagraph_capture`` + never refreshes the persistent ``state_indices_tensor_d`` buffer, so the + captured kernel reads the capture-time dummy slot (0) instead of the + request's real Mamba-cache slot -> garbage hidden states. + + Forcing ``treat_short_extends_as_decodes=True`` makes single-token extends + classify as decodes (``num_prefills==0``), which both matches the dispatched + FULL decode graph and re-enables the per-step ``state_indices_tensor_d`` + refresh. Multi-token context prefills (``query_len>1``) still classify as + prefills, so this is safe for mixed batches. Advancing Mamba state by one + token via the decode kernels is semantically identical to a 1-token prefill + chunk (it reads the slot's state and writes the advanced state back in + place), so no state update is lost — the only requirement is exactly one new + token per streamed step (``SamplingParams(max_tokens=1)``). + + Idempotent and process-global; the EasyMagpie plugin only ever serves this + model so the global patch is acceptable. + """ + orig = _mamba_attn.split_decodes_and_prefills + if getattr(orig, "_easymagpie_patched", False): + return + + def patched( + common_attn_metadata, + decode_threshold: int = 1, + require_uniform: bool = False, + treat_short_extends_as_decodes: bool = True, + ): + return orig( + common_attn_metadata, + decode_threshold=decode_threshold, + require_uniform=require_uniform, + treat_short_extends_as_decodes=True, + ) + + patched._easymagpie_patched = True + _mamba_attn.split_decodes_and_prefills = patched + logger.info("Mamba streaming-decode classification patch installed") + + class _SiluActivation(nn.Module): """``nn.Module`` wrapper around ``F.silu`` (so vLLM's NemotronHMLP can hold it).""" diff --git a/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/easymagpie.py b/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/easymagpie.py index 0b12af45faca..601d2b0c3465 100644 --- a/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/easymagpie.py +++ b/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/easymagpie.py @@ -65,12 +65,20 @@ appended), so callers never tokenize themselves. The resulting subword ids are consumed one per decode step (step ``k`` consumes id ``k``, embedded through the precomputed per-subword ``text_embedding`` table); once exhausted the text - channel is masked off. - - (Internal: the tokenized ids are stashed as ``text_tokens`` in the per-request - info dict between prefill and decode. A future streaming mode will let the - caller push subword ids gradually instead of one upfront ``text`` string; for - now assume ``text`` is always provided whole at prefill.) + channel is masked off. (Internal: the tokenized ids are stashed as + ``text_tokens`` in the per-request info dict between prefill and decode.) +* ``text_token`` (decode only, **streaming-text mode**) — when the caller omits + ``text`` at prefill, the request runs in streaming-text mode: the caller pushes + one subword id per decode step via ``additional_information`` under + ``text_token`` (a single int / 1-element tensor), embedded through the same + baked ``text_embedding`` table. This is the per-step counterpart to the whole + ``text`` string and is driven by vLLM-Omni's streaming-input API (an async + generator of ``StreamingInput`` chunks passed as the prompt, with + ``async_chunk=True``). Push the text-EOS id as the last real token; on any step + with no id (``text_token`` absent or ``< 0``, e.g. the sentinel ``-1``) the text + channel is masked off so the caller can keep pumping decode steps while the + audio tail finishes. Caller tokenization mirrors :meth:`_encode_text_stream` + (``tokenizer.encode(text, add_special_tokens=False) + [text_eos_id]``). * ``temperature`` / ``top_k`` (prefill only, optional) — audio sampling params for the local transformer. vLLM's ``SamplingParams.temperature`` drives only the dummy backbone token sampler, so the *audio* temperature/top-k are passed @@ -108,7 +116,11 @@ from vllm_omni.model_executor.models.output_templates import OmniOutput -from easymagpie_vllm_omni.backbone_patches import patch_moe_routed_scale, patch_silu_shared_experts +from easymagpie_vllm_omni.backbone_patches import ( + patch_mamba_streaming_decode, + patch_moe_routed_scale, + patch_silu_shared_experts, +) from easymagpie_vllm_omni.config import EasyMagpieOmniArch from easymagpie_vllm_omni.local_transformer import EasyMagpieCodePredictor @@ -123,7 +135,6 @@ # Context text used when the request omits ``context_text`` _DEFAULT_CONTEXT_TEXT = "[EN]" - # This class is not wrapped in ``@support_torch_compile``: the Nemotron-H # backbone and :class:`EasyMagpieCodePredictor` each manage their own # ``torch.compile`` / CUDA-graph capture internally, so the outer ``forward`` @@ -187,6 +198,12 @@ def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: # output is under-scaled by routed_scaling_factor. Restore it (no-op in # fp32/bf16 and when there are no MoE layers). patch_moe_routed_scale(self.backbone) + # The streaming-input path keeps extending the prompt, so vLLM's Mamba2 + # metadata builder would classify every single-token decode step as a + # prefill — breaking the FULL decode cudagraph (stale + # state_indices_tensor_d). Force single-token extends to classify as + # decodes so FULL/FULL_DECODE_ONLY cudagraphs read the right Mamba slot. + patch_mamba_streaming_decode() # ── Local transformer (its own compile group / CUDA graph) ────── with set_model_tag("local_transformer"): @@ -577,6 +594,34 @@ def _first_str(value: Any) -> str: return "" return str(value) + @staticmethod + def _coerce_opt_int(value: Any) -> Optional[int]: + """Best-effort extract a single int from a scalar / list / tensor / str. + + Used to read a per-step streamed ``text_token`` out of the request's + ``additional_information`` (which may wrap the id as a list, a 1-element + tensor, or a string depending on how the caller / transport packed it). + Returns ``None`` when no usable integer is present. + """ + if value is None: + return None + if isinstance(value, bool): # bool is an int subclass — handle explicitly. + return int(value) + if isinstance(value, int): + return value + if isinstance(value, float): + return int(value) + if isinstance(value, torch.Tensor): + return int(value.reshape(-1)[0].item()) if value.numel() > 0 else None + if isinstance(value, (list, tuple)): + return EasyMagpieTTSForConditionalGeneration._coerce_opt_int(value[0]) if value else None + if isinstance(value, str): + try: + return int(value.strip()) + except ValueError: + return None + return None + def preprocess( self, input_ids: torch.Tensor, @@ -662,10 +707,12 @@ def _preprocess_prefill( # Tokenize the caller's ``text`` in-model and stash the subword ids in the # per-request info dict (alongside the offsets) so each decode step # consumes one id from it without the caller ever running the tokenizer - # (see :meth:`_preprocess_decode`). The caller always passes ``text`` - # whole at prefill; a future streaming mode will instead let the caller - # push ``text_tokens`` ids gradually, which is why an already-present - # ``text_tokens`` list is left untouched here. + # (see :meth:`_preprocess_decode`). When the caller passes ``text`` whole + # at prefill we bake the ``text_tokens`` list here; an already-present + # ``text_tokens`` list is left untouched. When *neither* ``text`` nor + # ``text_tokens`` is provided the request runs in **streaming-text mode**: + # no list is baked, and :meth:`_preprocess_decode` instead reads one + # subword id per step from the streamed ``additional_information.text_token``. if not info_dict.get("text_tokens"): text = self._first_str(info_dict.get("text")) if text: @@ -818,17 +865,40 @@ def _preprocess_decode( decode_offset = int(info_dict.get("decode_offset", 0) or 0) info_update: dict[str, Any] = {"decode_offset": decode_offset + 1} - # ── Text channel ── (delay 0: one subword per step from step 0). Step k - # consumes text_tokens[k] (the list ends with the text eos id). Once the - # stream is exhausted the channel is masked off (adds nothing) rather than - # repeating the last token. The text stream leads the phoneme/audio - # streams by their respective delays. + # ── Text channel ── (delay 0: one subword per step from step 0). The text + # stream leads the phoneme/audio streams by their respective delays. Two + # mutually exclusive input modes are supported: + # + # * **Whole-text (non-streaming)** — the caller passed ``text`` whole at + # prefill; it was tokenized in-model and stashed as the ``text_tokens`` + # list (see :meth:`_preprocess_prefill`). Step k consumes + # ``text_tokens[k]`` (the list ends with the text-EOS id); once the + # stream is exhausted the channel is masked off (adds nothing) rather + # than repeating the last token. + # * **Streamed** — the caller did *not* pass ``text`` at prefill and + # instead pushes one subword id per decode step via + # ``additional_information`` under ``text_token`` (a single int / 1-elem + # tensor; close the stream by pushing the text-EOS id as the last real + # token). The model embeds that step's id and masks the channel off on + # any step that carries no id (``text_token`` absent or ``< 0``), so the + # caller can keep pumping decode steps after the text ends while the + # audio tail finishes. Because each streamed chunk overwrites the + # previous ``text_token`` in the per-request buffer, every step gets a + # fresh value (or the caller's sentinel ``-1`` to mask). text_tokens = info_dict.get("text_tokens") - if isinstance(text_tokens, list) and decode_offset < len(text_tokens): - self._dec_text_tokens[start] = int(text_tokens[decode_offset]) - self._dec_text_mask[start] = 1 + if isinstance(text_tokens, list): + if decode_offset < len(text_tokens): + self._dec_text_tokens[start] = int(text_tokens[decode_offset]) + self._dec_text_mask[start] = 1 + else: + self._dec_text_mask[start] = 0 else: - self._dec_text_mask[start] = 0 + streamed_id = self._coerce_opt_int(info_dict.get("text_token")) + if streamed_id is not None and streamed_id >= 0: + self._dec_text_tokens[start] = streamed_id + self._dec_text_mask[start] = 1 + else: + self._dec_text_mask[start] = 0 # ── Phoneme channel ── opens at decode step == ``phonemes_delay`` (seeded # with phoneme BOS), then feeds back the previous step's prediction, and From 87232e6840218594d9316499f6777f1a8e5cb955 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimkov Date: Tue, 9 Jun 2026 23:50:09 +0200 Subject: [PATCH 29/45] xamples/tts/easymagpie_vllm_omni/easymagpie_inference_demo.ipynb: add demo for streaming text tokens Signed-off-by: Viacheslav Klimkov --- .../easymagpie_inference_demo.ipynb | 1208 ++++++++--------- 1 file changed, 555 insertions(+), 653 deletions(-) diff --git a/examples/tts/easymagpie_vllm_omni/easymagpie_inference_demo.ipynb b/examples/tts/easymagpie_vllm_omni/easymagpie_inference_demo.ipynb index 2c773c51fbfe..65615ea2d2ee 100644 --- a/examples/tts/easymagpie_vllm_omni/easymagpie_inference_demo.ipynb +++ b/examples/tts/easymagpie_vllm_omni/easymagpie_inference_demo.ipynb @@ -1,656 +1,558 @@ { - "cells": [ - { - "cell_type": "markdown", - "id": "d5a1129d", - "metadata": {}, - "source": [ - "# EasyMagpieTTS — vLLM-Omni inference demo\n", - "\n", - "This notebook runs an end-to-end inference pass through the\n", - "[`easymagpie_vllm_omni`](./easymagpie_vllm_omni) model definition using a\n", - "**converted checkpoint directory** produced by\n", - "[`easy_magpietts_convert_to_vllm.py`](./easy_magpietts_convert_to_vllm.py)\n", - "(weights + `config.json` + text tokenizer + speaker embeddings). It exercises the\n", - "full engine path: prefill -> autoregressive decode -> audio-code extraction.\n", - "\n", - "It follows the same `AsyncOmni` single-stage pattern as the reference\n", - "`qwen3-tts` and `eartts` demos:\n", - "\n", - "* **prefill** — the caller supplies the speaker-encoded context-audio embedding\n", - " via `additional_information.speaker_embedding` `(T_audio, embedding_dim)` plus a\n", - " plain `context_text` string; the model assembles the full prefill context\n", - " (`[task_embedding? | speaker_embedding | context_text_embedded]`) and tokenizes\n", - " `context_text` itself. `prompt_token_ids = [0] * prompt_len`, sized with\n", - " `EasyMagpieTTSForConditionalGeneration.estimate_prompt_len(...)`.\n", - "* **decode** — the caller passes the plain target sentence as\n", - " `additional_information.text`; the model tokenizes it in-engine (no caller-side\n", - " tokenization) and consumes one subword id per step. The local transformer\n", - " samples all `C * S` stacked audio codebooks for the frame.\n", - "* **output** — per-step audio codes are surfaced on\n", - " `OmniOutput.multimodal_outputs[\\\"audio_codes\\\"]` (`BT x num_codebooks`), and the\n", - " engine accumulates them across steps just like eartts, so we trim to the last\n", - " `len(token_ids)` decoded rows.\n", - "\n", - "> **Converted checkpoint.** Set `MODEL_DIR` below to the directory written by the\n", - "> converter. The engine reads the `config.json`, weights, and tokenizer straight\n", - "> from it — no hardcoded config, no dummy weights.\n", - "\n", - "> **Environment.** Run this inside the bootstrapped `vllm_omni_env` (vLLM +\n", - "> vLLM-Omni + compatible torch) with the plugin installed:\n", - "> ```bash\n", - "> source /path/to/vllm_omni_env/bin/activate\n", - "> pip install -e examples/tts/easymagpie_vllm_omni\n", - "> ```" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c9a71b74", - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "\n", - "# Single-process executor below, but keep spawn semantics consistent with the\n", - "# qwen3-tts / eartts demos in case you switch to a multiproc backend.\n", - "os.environ.setdefault(\"VLLM_WORKER_MULTIPROC_METHOD\", \"spawn\")\n", - "\n", - "import json\n", - "import tempfile\n", - "import uuid\n", - "from pathlib import Path\n", - "\n", - "import torch\n", - "import yaml\n", - "\n", - "from vllm import SamplingParams\n", - "from vllm.sampling_params import RequestOutputKind\n", - "from vllm_omni import AsyncOmni\n", - "\n", - "# Importing the model package is optional (the engine resolves the arch via the\n", - "# `vllm.general_plugins` entry point installed with the package), but doing it\n", - "# here surfaces the arch dataclass we use to read scalars from the config.\n", - "from easymagpie_vllm_omni.config import EasyMagpieOmniArch\n", - "\n", - "print(\"torch:\", torch.__version__, \"| cuda:\", torch.cuda.is_available())" - ] - }, - { - "cell_type": "markdown", - "id": "f7ff55fe", - "metadata": {}, - "source": [ - "## 1. Point at the converted model directory\n", - "\n", - "Set `MODEL_DIR` to the directory written by\n", - "[`easy_magpietts_convert_to_vllm.py`](./easy_magpietts_convert_to_vllm.py). It\n", - "already contains everything the engine needs:\n", - "\n", - "* `config.json` — the registered arch + Nemotron-H backbone scalars + EasyMagpie\n", - " scalars (read by `EasyMagpieOmniArch.from_hf_config`),\n", - "* `model.safetensors` — the converted weights (backbone + TTS submodules + the\n", - " baked per-subword `text_embedding` table),\n", - "* the text-conditioning tokenizer (`tokenizer.json` / `tokenizer_config.json`),\n", - " loaded in-engine to tokenize the per-request `context_text`,\n", - "* `speaker_embeddings/.pt` — pre-computed speaker embeddings for reference\n", - " voices.\n", - "\n", - "The backbone is a **Nemotron-H** hybrid (Mamba2 + attention + MoE) decoder:\n", - "`EasyMagpieTTSForConditionalGeneration` constructs vLLM's `NemotronHModel` and\n", - "implements the hybrid-Mamba interfaces (`HasInnerState` / `IsHybrid` /\n", - "`SupportsMambaPrefixCaching`), exactly like the EasyMagpie vLLM *sidecar*. The\n", - "phoneme branch is enabled in the converted config; the model self-predicts\n", - "phonemes, so no phoneme stream needs to be supplied in the prompt.\n", - "\n", - "We just read the `config.json` here to surface a few scalars used for building\n", - "the prompt (`text_vocab_size`, the audio EOS id, whether a task embedding exists)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3e0df89e", - "metadata": {}, - "outputs": [], - "source": [ - "# Directory produced by easy_magpietts_convert_to_vllm.py.\n", - "MODEL_DIR = Path(\"/home/vklimkov/workspace/emp/NeMo/examples/tts/easymagpie_vllm_omni/easymp_vllm_model\")\n", - "assert (MODEL_DIR / \"config.json\").exists(), f\"No config.json under {MODEL_DIR}; run the converter first.\"\n", - "\n", - "# Read the converted config to surface a few scalars used when building the\n", - "# prompt. The engine itself loads everything from MODEL_DIR; we only peek here.\n", - "config = json.loads((MODEL_DIR / \"config.json\").read_text())\n", - "arch = EasyMagpieOmniArch.from_hf_config(type(\"Cfg\", (), config))\n", - "\n", - "# Subword id space of the baked text-embedding table (the streaming text stream\n", - "# indexes into it). The model's text BOS/EOS/CFG-UNK ids are the last 3 rows.\n", - "TEXT_VOCAB = int(config[\"text_vocab_size\"])\n", - "TEXT_EOS_ID = TEXT_VOCAB - 2 # matches EasyMagpieTTSInferenceModel.eos_id\n", - "\n", - "AUDIO_STOP_TOKEN_ID = max(1, int(config.get(\"vocab_size\", 2)) - 1)\n", - "\n", - "print(f\"Model dir : {MODEL_DIR}\")\n", - "print(f\"embedding_dim : {arch.embedding_dim}\")\n", - "print(f\"num_stacked_codebooks : {arch.num_stacked_codebooks} (C*S)\")\n", - "print(f\"tokens / codebook : {arch.num_all_tokens_per_codebook} (codebook_size + specials)\")\n", - "print(f\"audio_bos / audio_eos id : {arch.audio_bos_id} / {arch.audio_eos_id}\")\n", - "print(f\"text_vocab / text_eos : {TEXT_VOCAB} / {TEXT_EOS_ID}\")\n", - "print(f\"audio-EOS stop token id : {AUDIO_STOP_TOKEN_ID}\")" - ] - }, - { - "cell_type": "markdown", - "id": "012df58d", - "metadata": {}, - "source": [ - "## 2. Single-stage `AsyncOmni` engine\n", - "\n", - "A single `llm` stage that runs the EasyMagpie talker, mirroring the eartts demo\n", - "(`worker_type=\\\"ar\\\"`, `OmniARScheduler`). The stage declares\n", - "`engine_output_type=\\\"audio\\\"` / `final_output_type=\\\"audio\\\"`: for a single-stage\n", - "AR TTS model these make the runner attach the per-step `audio_codes` multimodal\n", - "payload to the output (with `\\\"latent\\\"` the payload is dropped because nothing\n", - "downstream consumes it, and `multimodal_output[\\\"audio_codes\\\"]` comes back\n", - "`None`).\n", - "\n", - "`skip_tokenizer_init: true` — we feed `prompt_token_ids` directly, so vLLM\n", - "doesn't need its own tokenizer for the prompt. The model loads the bundled\n", - "`AutoTokenizer` from `MODEL_DIR` in-engine and uses it to tokenize both\n", - "`context_text` and the target `text`.\n", - "\n", - "`max_model_len` must cover `T_ctx` (prefill) + the number of decode steps." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5085e9a4", - "metadata": {}, - "outputs": [], - "source": [ - "DECODE_STEPS = 256 # max number of audio frames to decode (trimmed at audio EOS)\n", - "# Prefill length is derived at prompt-build time from the speaker embedding +\n", - "# tokenized context_text (see the prompt cell); these just need to be large\n", - "# enough to cover prefill + decode.\n", - "MAX_MODEL_LEN = 1024\n", - "MAX_NUM_BATCHED_TOKENS = 1024\n", - "\n", - "stage_cfg = {\n", - " \"stage_args\": [\n", - " {\n", - " \"stage_id\": 0,\n", - " \"stage_type\": \"llm\",\n", - " \"is_comprehension\": True,\n", - " \"final_output\": True,\n", - " # \"audio\" (not \"latent\") is required for a single-stage AR TTS model:\n", - " # it makes the AR model runner attach the per-step multimodal payload\n", - " # (\"audio_codes\") to the EngineCoreOutput even though no downstream\n", - " # stage consumes it, so the codes reach the client. With \"latent\" the\n", - " # payload is dropped and multimodal_output[\"audio_codes\"] is None.\n", - " \"final_output_type\": \"audio\",\n", - " \"runtime\": {\"devices\": \"0\"},\n", - " \"engine_args\": {\n", - " \"model_stage\": \"easymagpie\",\n", - " \"max_num_seqs\": 1,\n", - " \"model_arch\": \"EasyMagpieTTSForConditionalGeneration\",\n", - " \"worker_type\": \"ar\",\n", - " \"scheduler_cls\": \"vllm_omni.core.sched.omni_ar_scheduler.OmniARAsyncScheduler\",\n", - " #\"enforce_eager\": True, # dummy run: skip CUDA-graph capture for a faster start\n", - " \"trust_remote_code\": True,\n", - " \"async_scheduling\": True,\n", - " \"enable_prefix_caching\": False,\n", - " \"engine_output_type\": \"audio\",\n", - " \"gpu_memory_utilization\": 0.8,\n", - " \"distributed_executor_backend\": \"uni\",\n", - " \"max_num_batched_tokens\": MAX_NUM_BATCHED_TOKENS,\n", - " \"max_model_len\": MAX_MODEL_LEN,\n", - " # bf16 (not fp32): the Nemotron-H fused-MoE Triton kernel's block\n", - " # sizes are tuned for 16-bit and overflow shared memory in fp32.\n", - " #\"dtype\": \"bfloat16\",\n", - " \"dtype\": \"float16\",\n", - " \"mamba_ssm_cache_dtype\": \"float32\",\n", - " \"attention_backend\": \"TRITON_ATTN\",\n", - " # We feed prompt_token_ids directly; the model loads the bundled\n", - " # AutoTokenizer from MODEL_DIR to tokenize context_text + text.\n", - " \"skip_tokenizer_init\": True,\n", - " },\n", - " \"default_sampling_params\": {\n", - " \"temperature\": 0.0,\n", - " \"max_tokens\": DECODE_STEPS,\n", - " \"detokenize\": False,\n", - " # model forwards EOS to dummy output tokens\n", - " \"ignore_eos\": True,\n", - " \"stop_token_ids\": [AUDIO_STOP_TOKEN_ID],\n", - " },\n", - " }\n", - " ],\n", - "}\n", - "\n", - "_tmp = tempfile.NamedTemporaryFile(\n", - " mode=\"w\", suffix=\".yaml\", prefix=\"easymagpie_omni_demo_\", delete=False,\n", - ")\n", - "yaml.dump(stage_cfg, _tmp, sort_keys=False)\n", - "_tmp.close()\n", - "STAGE_CFG_PATH = _tmp.name\n", - "print(f\"Stage config: {STAGE_CFG_PATH}\")\n", - "\n", - "omni = AsyncOmni(\n", - " model=str(MODEL_DIR),\n", - " stage_configs_path=STAGE_CFG_PATH,\n", - " log_stats=False,\n", - " stage_init_timeout=300,\n", - ")\n", - "print(\"Engine ready (single stage: EasyMagpie talker)\")" - ] - }, - { - "cell_type": "markdown", - "id": "2736b86d", - "metadata": {}, - "source": [ - "## 3. Build the prompt\n", - "\n", - "Per-request input, passed through `additional_information`:\n", - "\n", - "* **`speaker_embedding`** `(T_audio, embedding_dim)` — the speaker-encoded\n", - " context-audio embedding (the audio branch of `prepare_context_tensors`),\n", - " loaded here from `MODEL_DIR/speaker_embeddings/.pt` (written by\n", - " the converter). The model assembles the full prefill context itself as\n", - " `[task_embedding? | speaker_embedding | context_text_embedded]`.\n", - "* **`context_text`** — a plain conditioning string, here `\"[EN]\"`. The model\n", - " tokenizes it in-engine and embeds it through the baked `text_embedding` table.\n", - "* **`text`** — the plain target sentence to synthesize. The model tokenizes it\n", - " in-engine at prefill (HF special tokens disabled, trailing text-EOS id\n", - " appended — matching the reference `encode(transcript) + [eos_id]`) and streams\n", - " one subword id per decode step; once exhausted the channel is masked off. No\n", - " caller-side tokenization needed.\n", - "\n", - "`prompt_token_ids = [0] * prompt_len` are placeholders (the model feeds the\n", - "backbone via `inputs_embeds`, never these ids). `prompt_len` must equal the\n", - "assembled context length, so we size it with the model's\n", - "`estimate_prompt_len(...)` — the length-only mirror of the in-engine prefill\n", - "assembly (à la qwen3-tts's `estimate_prompt_len_from_additional_information`).\n", - "\n", - "(If the checkpoint had a phoneme branch you'd also stream `phoneme_tokens`.)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "697c74b3", - "metadata": {}, - "outputs": [], - "source": [ - "torch.manual_seed(0)\n", - "\n", - "from transformers import AutoTokenizer\n", - "\n", - "from easymagpie_vllm_omni.easymagpie import EasyMagpieTTSForConditionalGeneration\n", - "\n", - "# Speaker-encoded context audio (audio branch of prepare_context_tensors),\n", - "# pre-computed by the converter into MODEL_DIR/speaker_embeddings/.pt.\n", - "SPEAKER_NAME = \"eng\"\n", - "_loaded = torch.load(MODEL_DIR / \"speaker_embeddings\" / f\"{SPEAKER_NAME}.pt\", map_location=\"cpu\")\n", - "speaker_embedding = _loaded[\"speaker_encoding\"] if isinstance(_loaded, dict) else _loaded\n", - "speaker_embedding = speaker_embedding.to(torch.float32)\n", - "\n", - "# Plain conditioning string; the model tokenizes + embeds it in-engine.\n", - "CONTEXT_TEXT = \"[EN]\"\n", - "\n", - "# Target sentence to synthesize.\n", - "TEXT = \"Hello, this is a test of the EasyMagpie text to speech model.\"\n", - "\n", - "# Same tokenizer the engine loads from MODEL_DIR. Used only to size the prefill\n", - "# placeholders so prompt_token_ids length matches the assembled context (the\n", - "# target text is tokenized in-engine — we just pass the plain string below).\n", - "tokenizer = AutoTokenizer.from_pretrained(MODEL_DIR, trust_remote_code=True)\n", - "prompt_len = EasyMagpieTTSForConditionalGeneration.estimate_prompt_len(\n", - " speaker_embedding,\n", - " tokenize=lambda t: tokenizer.encode(t),\n", - " context_text=CONTEXT_TEXT,\n", - " has_task_embedding=arch.num_task_embeddings > 0,\n", - ")\n", - "\n", - "# Audio (local-transformer) sampling params. vLLM's SamplingParams.temperature\n", - "# drives only the dummy backbone token sampler, so the *audio* temperature/top-k\n", - "# are forwarded via additional_information. temperature=0.0 == argmax\n", - "# (deterministic; matches the torch reference run with --temperature 0.0 --no_cfg).\n", - "LT_TEMPERATURE = 0.0\n", - "LT_TOPK = 80\n", - "\n", - "additional_information = {\n", - " \"speaker_embedding\": speaker_embedding, # (T_audio, embedding_dim) tensor\n", - " \"context_text\": CONTEXT_TEXT, # plain string, tokenized in-model\n", - " \"text\": TEXT, # plain target sentence, tokenized in-model\n", - " \"temperature\": LT_TEMPERATURE, # audio sampling temperature (local transformer)\n", - " \"top_k\": LT_TOPK, # audio sampling top-k (local transformer)\n", - "}\n", - "\n", - "prompt = {\n", - " \"prompt_token_ids\": [0] * prompt_len, # prefill placeholders\n", - " \"additional_information\": additional_information,\n", - "}\n", - "\n", - "assert prompt_len + DECODE_STEPS <= MAX_MODEL_LEN, (\n", - " f\"prompt_len ({prompt_len}) + decode steps ({DECODE_STEPS}) exceeds \"\n", - " f\"MAX_MODEL_LEN ({MAX_MODEL_LEN}); raise MAX_MODEL_LEN / MAX_NUM_BATCHED_TOKENS.\"\n", - ")\n", - "\n", - "print(f\"speaker_embedding : {tuple(speaker_embedding.shape)}\")\n", - "print(f\"context_text : {CONTEXT_TEXT!r} -> {tokenizer.encode(CONTEXT_TEXT)}\")\n", - "print(f\"text : {TEXT!r} (tokenized in-engine)\")\n", - "print(f\"prompt_len (placeholders) : {prompt_len}\")\n", - "print(f\"decode steps (max_tokens) : {DECODE_STEPS}\")\n", - "\n", - "sampling_params = SamplingParams(\n", - " temperature=0.0, # backbone token sampler is a no-op (audio is sampled in the local transformer)\n", - " max_tokens=DECODE_STEPS,\n", - " detokenize=False,\n", - " ignore_eos=True, # audio EOS lives in the codes, not the vLLM token stream\n", - " # The model emits AUDIO_STOP_TOKEN_ID on the backbone stream at the EOS frame\n", - " # (audio EOS in any codebook), so vLLM ends the request there instead of\n", - " # decoding the full DECODE_STEPS budget. stop_token_ids is honored regardless\n", - " # of ignore_eos.\n", - " stop_token_ids=[AUDIO_STOP_TOKEN_ID],\n", - " # DELTA: expose audio_codes as a *growing list* of per-step frame deltas during\n", - " # decode (element 0 = prefill prefix, element 1+d = decode frame d) instead of\n", - " # re-sending the whole cumulative tensor each step. This is the streaming\n", - " # consumption pattern used by the Triton deployment (model_repository/easymp).\n", - " output_kind=RequestOutputKind.DELTA,\n", - ")" - ] - }, - { - "cell_type": "markdown", - "id": "3ef8934d", - "metadata": {}, - "source": [ - "## 4. Run inference and extract audio codes (delta streaming)\n", - "\n", - "`omni.generate(...)` is an async generator yielding one `RequestOutput` per\n", - "engine step. Because we set `output_kind=RequestOutputKind.DELTA`, the engine\n", - "exposes `multimodal_output[\\\"audio_codes\\\"]` as a **growing list** during decode —\n", - "`[prefill_prefix, frame_0, frame_1, ...]` — appending one `(1, C*S)` frame per\n", - "step (element 0 is the prefill prefix, element `1 + d` is decode frame `d`). So\n", - "each step we collect just the *new* frames since the previous step — exactly the\n", - "per-step delta a streaming client would push to the codec frame-by-frame (this is\n", - "the same pattern the Triton deployment in `model_repository/easymp` uses). The\n", - "prefill (first) and final (consolidated) steps yield a cumulative *tensor*\n", - "instead; we keep the final consolidated tensor as the authoritative full sequence\n", - "for vocoding. We then drop the leading `speech_delay` warm-up frames and trim the\n", - "trailing audio-EOS frame (the model signals end-of-speech in the codes, not in the\n", - "vLLM token stream)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6d0ccbd4", - "metadata": {}, - "outputs": [], - "source": [ - "async def run_request(prompt: dict, sampling_params):\n", - " \"\"\"Stream the request, collecting the per-step audio-code *deltas*.\n", - "\n", - " With ``RequestOutputKind.DELTA`` the engine exposes ``audio_codes`` as a\n", - " *growing list* during decode — ``[prefill_prefix, frame_0, frame_1, ...]`` —\n", - " appending one ``(1, C*S)`` frame per step (the prefill and final steps yield a\n", - " cumulative tensor instead). We collect each new frame as it arrives (exactly\n", - " what a streaming client would push to the codec frame-by-frame) and also keep\n", - " the final consolidated tensor as the authoritative full sequence for vocoding.\n", - " \"\"\"\n", - " request_id = f\"easymagpie-{uuid.uuid4().hex[:8]}\"\n", - " delta_frames = [] # per-step decode-frame deltas (excludes prefill prefix)\n", - " prev_frames = 0 # decode frames already collected from the growing list\n", - " final_tensor = None # last consolidated (T_ctx + N, C*S) tensor observed\n", - " saw_list = False # whether audio_codes ever appeared in the growing-list form\n", - " cur_num_tokens = 0\n", - " num_steps = 0\n", - " final_ro = None\n", - " async for stage_output in omni.generate(\n", - " prompt,\n", - " sampling_params_list=[sampling_params],\n", - " request_id=request_id,\n", - " ):\n", - " final_ro = stage_output\n", - " num_steps += 1\n", - "\n", - " co = stage_output.outputs[0] if stage_output.outputs else None\n", - " cum_ids = getattr(co, \"cumulative_token_ids\", None) if co else None\n", - " if cum_ids is not None:\n", - " cur_num_tokens = len(cum_ids)\n", - " elif co is not None:\n", - " cur_num_tokens = len(getattr(co, \"token_ids\", []) or [])\n", - "\n", - " payload = (stage_output.multimodal_output or {}).get(\"audio_codes\")\n", - " if isinstance(payload, list):\n", - " # DELTA decode form: element 0 is the prefill prefix; decode frame d is\n", - " # at index 1 + d. Collect frames new since the previous step.\n", - " saw_list = True\n", - " avail = len(payload) - 1\n", - " if avail > prev_frames:\n", - " for t in payload[1 + prev_frames : 1 + avail]:\n", - " if isinstance(t, torch.Tensor):\n", - " delta_frames.append(t.detach().cpu().to(torch.long))\n", - " prev_frames = avail\n", - " elif isinstance(payload, torch.Tensor) and cur_num_tokens > 0:\n", - " # prefill (first) or consolidated (final) yield: the last cur_num_tokens\n", - " # rows are the decode frames (the T_ctx prefill prefix rows lead).\n", - " final_tensor = payload[-cur_num_tokens:].detach().cpu().to(torch.long)\n", - "\n", - " return final_ro, num_steps, delta_frames, final_tensor, saw_list\n", - "\n", - "\n", - "final_ro, num_steps, delta_frames, final_tensor, saw_list = await run_request(prompt, sampling_params)\n", - "assert final_ro is not None, \"no output from engine\"\n", - "\n", - "print(f\"Engine steps yielded : {num_steps}\")\n", - "print(\n", - " f\"Per-step delta frames seen : {len(delta_frames)} \"\n", - " f\"(audio_codes exposed as a {'growing list' if saw_list else 'tensor'})\"\n", - ")\n", - "\n", - "# Prefer the authoritative consolidated tensor (it includes the trailing EOS\n", - "# frame); fall back to the streamed deltas if no consolidated tensor was observed.\n", - "if final_tensor is not None:\n", - " audio_codes = final_tensor\n", - "elif delta_frames:\n", - " audio_codes = torch.cat(delta_frames, dim=0)\n", - "else:\n", - " audio_codes = None\n", - "\n", - "if isinstance(audio_codes, torch.Tensor):\n", - " source = \"final consolidated tensor\" if final_tensor is not None else \"streamed deltas\"\n", - " print(f\"audio_codes shape (decode) : {tuple(audio_codes.shape)} (from {source})\")\n", - "\n", - " # Drop the leading streaming_speech_delay warm-up frames. With the streaming\n", - " # delay the audio stream only opens at decode step == speech_delay, so the\n", - " # first speech_delay decoded frames carry no real audio (the audio channel was\n", - " # masked off while the model consumed lookahead text/phonemes).\n", - " speech_delay = int(getattr(arch, \"streaming_speech_delay\", 0) or 0)\n", - " if speech_delay > 0:\n", - " print(f\"dropping {speech_delay} leading speech-delay warm-up frames\")\n", - " audio_codes = audio_codes[speech_delay:].contiguous()\n", - "\n", - " # Trim the trailing audio-EOS frame. The engine stops the request the moment\n", - " # the backbone emits AUDIO_STOP_TOKEN_ID (driven high at the audio-EOS frame),\n", - " # so when it finished for that reason the *last* decoded frame is the EOS frame\n", - " # itself — its codes carry audio_eos_id and must not be vocoded.\n", - " # NOTE: we actually expect EOS to be emited\n", - " co = final_ro.outputs[0] if final_ro.outputs else None\n", - " finish_reason = getattr(co, \"finish_reason\", None)\n", - " stop_reason = getattr(co, \"stop_reason\", None)\n", - " print(f\"finish_reason / stop_reason : {finish_reason} / {stop_reason}\")\n", - " if finish_reason == \"stop\" and stop_reason == AUDIO_STOP_TOKEN_ID and audio_codes.shape[0] > 0:\n", - " print(f\"dropping trailing audio-EOS frame at {audio_codes.shape[0] - 1}\")\n", - " audio_codes = audio_codes[:-1].contiguous()\n", - " else:\n", - " print(f\"no engine EOS stop (finish_reason={finish_reason}); using full decode\")\n", - "\n", - " print(f\"audio_codes shape (final) : {tuple(audio_codes.shape)}\")\n", - " print(f\"audio_codes dtype : {audio_codes.dtype}\")\n", - " print(f\"codes min / max : {int(audio_codes.min())} / {int(audio_codes.max())}\")\n", - " print(f\"first frame codes : {audio_codes[0].tolist()}\")\n", - "else:\n", - " audio_codes = None\n", - " print(\"audio_codes : no codes collected\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "04196662", - "metadata": {}, - "outputs": [], - "source": [ - "import matplotlib.pylab as plt\n", - "\n", - "plt.imshow(audio_codes.T, aspect=\"auto\")\n", - "plt.colorbar()\n", - "plt.show()" - ] - }, - { - "cell_type": "markdown", - "id": "a32b07d5", - "metadata": {}, - "source": [ - "## 5. Decode audio codes to a waveform\n", - "\n", - "The engine emits **stacked** codebooks: `audio_codes` is `(T, C*S)` where\n", - "`C = num_audio_codebooks` and `S = frame_stacking_factor` (here `C*S = 16`).\n", - "To turn them back into a waveform we mirror what\n", - "[`EasyMagpieTTSInferenceModel.streaming_finalize`](../../../nemo/collections/tts/models/easy_magpietts_inference.py)\n", - "does at the end of inference:\n", - "\n", - "1. **load the codec** — the `.nemo` audio codec used to train the model\n", - " (`AudioCodecModel.restore_from(...)`, discriminator dropped to save memory),\n", - "2. **unstack** `(T, C*S)` -> `(1, C, T*S)` — the inverse of `stack_codes`,\n", - "3. **convert codec tokens** — this checkpoint was trained on a *regrouped* FSQ\n", - " index space (8 codebooks of size 1024) that differs from the codec's native\n", - " `GroupFiniteScalarQuantizer` (4 codebooks), so we map the model's tokens back\n", - " to the codec's space (`convert_new_to_original`) before decoding. The\n", - " `vector_quantizer` config is read straight from the source EasyMagpie `.nemo`\n", - " (config only, no weights). If the two spaces match, this step is skipped.\n", - "4. **decode** `codec_model.decode(tokens=..., tokens_len=...)` -> waveform at\n", - " `codec_model.output_sample_rate`.\n", - "\n", - "> Set `CODEC_MODEL_PATH` / `EASYMAGPIE_NEMO` to the **same** `.nemo` files passed\n", - "> to `easy_magpietts_convert_to_vllm.py` (`--codec_model_path` / `--nemo_file`).\n", - "> This step needs NeMo importable in the current environment." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "aa57a573", - "metadata": {}, - "outputs": [], - "source": [ - "from hydra.utils import instantiate\n", - "from IPython.display import Audio, display\n", - "\n", - "from nemo.collections.tts.models import AudioCodecModel\n", - "from nemo.collections.tts.models.easy_magpietts_inference import EasyMagpieTTSInferenceModel\n", - "from nemo.collections.tts.modules.audio_codec_modules import VectorQuantizerIndexConverter\n", - "\n", - "# Same .nemo codec passed to easy_magpietts_convert_to_vllm.py --codec_model_path.\n", - "CODEC_MODEL_PATH = \"/home/vklimkov/workspace/emp/ckpt/easymagpietts_NEXT/25fps_spectral_codec_with_bandwidth_extension.nemo\"\n", - "\n", - "# --- load the codec once (drop the discriminator to save memory) ---\n", - "_codec_cfg = AudioCodecModel.restore_from(CODEC_MODEL_PATH, return_config=True)\n", - "if \"use_scl_loss\" in _codec_cfg:\n", - " _codec_cfg.use_scl_loss = False\n", - "codec_model = AudioCodecModel.restore_from(\n", - " CODEC_MODEL_PATH, strict=False, override_config_path=_codec_cfg\n", - ")\n", - "if hasattr(codec_model, \"discriminator\"):\n", - " del codec_model.discriminator\n", - "codec_model = codec_model.eval().to(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", - "codec_device = next(codec_model.parameters()).device\n", - "\n", - "# Source EasyMagpie .nemo (--nemo_file). Only its config is read (no weights),\n", - "# to recover the `vector_quantizer` override the model was trained with.\n", - "EASYMAGPIE_NEMO = \"/home/vklimkov/workspace/emp/ckpt/easymagpietts_NEXT/2605_NemotronTTS_V0.2/v2/2605_EMTTS_SmallMamba_Step150k_posttrained_epoch12.nemo\"\n", - "\n", - "# --- optional codec-token converter ---------------------------------------\n", - "# EasyMagpie can be trained on a *regrouped* FSQ index space (here C=8 codebooks\n", - "# of size 1024) that differs from the codec's native quantizer (this codec's\n", - "# GroupFiniteScalarQuantizer has 4 codebooks). When they differ the model's\n", - "# tokens must be mapped back to the codec's space before decoding, exactly as\n", - "# EasyMagpieTTSInferenceModel does via `_codec_converter` /\n", - "# `CodecHelper.codes_to_audio`.\n", - "_em_cfg = EasyMagpieTTSInferenceModel.restore_from(EASYMAGPIE_NEMO, return_config=True)\n", - "_vq_cfg = _em_cfg.get(\"vector_quantizer\")\n", - "if _vq_cfg is not None and instantiate(_vq_cfg).num_codebooks != codec_model.vector_quantizer.num_codebooks:\n", - " codec_converter = VectorQuantizerIndexConverter(\n", - " vector_quantizer_original=codec_model.vector_quantizer,\n", - " vector_quantizer_new=instantiate(_vq_cfg),\n", - " ).to(codec_device)\n", - "else:\n", - " codec_converter = None\n", - "print(f\"codec native codebooks : {codec_model.vector_quantizer.num_codebooks}\")\n", - "print(f\"codec token converter : {'enabled' if codec_converter is not None else 'not needed'}\")\n", - "\n", - "S = arch.frame_stacking_factor # stacking factor (sub-frames per stacked frame)\n", - "C = arch.num_stacked_codebooks // S # real codec codebooks\n", - "assert audio_codes.dim() == 2 and audio_codes.size(1) == arch.num_stacked_codebooks, (\n", - " f\"expected audio_codes (T, {arch.num_stacked_codebooks}); got {tuple(audio_codes.shape)}\"\n", - ")\n", - "\n", - "# --- unstack (T, C*S) -> (1, C, T*S): inverse of EasyMagpie stack_codes ---\n", - "stacked = audio_codes.to(codec_device, torch.long).T.unsqueeze(0) # (1, C*S, T)\n", - "T_out = stacked.size(-1)\n", - "codes = stacked.view(1, C, S, T_out).permute(0, 1, 3, 2).reshape(1, C, T_out * S) # (1, C, T*S)\n", - "codes_len = torch.tensor([codes.size(-1)], device=codec_device, dtype=torch.long)\n", - "\n", - "# Pad very short sequences (codec needs a few frames), matching _prepare_codes_for_decode.\n", - "MIN_LEN = 4\n", - "if int(codes_len.min()) < MIN_LEN:\n", - " codes = torch.nn.functional.pad(codes, (0, MIN_LEN - int(codes_len.min())), value=0)\n", - " codes_len = codes_len.clamp(min=MIN_LEN)\n", - "\n", - "# Drop any stray special tokens (BOS/EOS/MASK live at codebook_size..) so every\n", - "# index is a valid codec entry before decoding.\n", - "codes = codes.clamp_(0, arch.codebook_size - 1)\n", - "\n", - "# --- decode codes -> waveform (mirrors CodecHelper.codes_to_audio) ---\n", - "with torch.no_grad(), torch.autocast(device_type=codec_device.type, dtype=torch.float32):\n", - " if codec_converter is not None:\n", - " codes = codec_converter.convert_new_to_original(audio_tokens=codes, audio_lens=codes_len)\n", - " audio, audio_len = codec_model.decode(tokens=codes, tokens_len=codes_len)\n", - "\n", - "waveform = audio[0, : int(audio_len[0])].detach().cpu().float().numpy()\n", - "sample_rate = int(codec_model.output_sample_rate)\n", - "\n", - "print(f\"codes (unstacked) shape : {tuple(codes.shape)} (1, C={C}, T*S={codes.size(-1)})\")\n", - "print(f\"waveform samples : {waveform.shape[0]} ({waveform.shape[0] / sample_rate:.2f}s @ {sample_rate} Hz)\")\n", - "\n", - "display(Audio(waveform, rate=sample_rate))" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "emp", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.13" - } + "cells": [ + { + "cell_type": "markdown", + "id": "d5a1129d", + "metadata": {}, + "source": [ + "# EasyMagpieTTS — vLLM-Omni inference demo\n", + "\n", + "End-to-end inference through `easymagpie_vllm_omni` on a checkpoint converted by\n", + "`easy_magpietts_convert_to_vllm.py`: prefill (speaker embedding + context text) ->\n", + "autoregressive decode (target text, one subword per step) -> stacked audio codes ->\n", + "vocode to waveform.\n", + "\n", + "Run inside the `vllm_omni_env` with the plugin installed\n", + "(`pip install -e examples/tts/easymagpie_vllm_omni`)." + ] }, - "nbformat": 4, - "nbformat_minor": 5 + { + "cell_type": "code", + "execution_count": null, + "id": "c9a71b74", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "os.environ.setdefault(\"VLLM_WORKER_MULTIPROC_METHOD\", \"spawn\")\n", + "\n", + "import json\n", + "import tempfile\n", + "import uuid\n", + "from pathlib import Path\n", + "\n", + "import torch\n", + "import yaml\n", + "\n", + "from vllm import SamplingParams\n", + "from vllm.sampling_params import RequestOutputKind\n", + "from vllm_omni import AsyncOmni\n", + "\n", + "from easymagpie_vllm_omni.config import EasyMagpieOmniArch\n", + "\n", + "print(\"torch:\", torch.__version__, \"| cuda:\", torch.cuda.is_available())" + ] + }, + { + "cell_type": "markdown", + "id": "f7ff55fe", + "metadata": {}, + "source": [ + "## 1. Converted model directory\n", + "\n", + "`MODEL_DIR` is the directory written by the converter: `config.json`, weights, the\n", + "text tokenizer, and per-speaker embeddings. The engine loads everything from it;\n", + "here we only read a few config scalars used to build the prompt." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3e0df89e", + "metadata": {}, + "outputs": [], + "source": [ + "MODEL_DIR = Path(\"/home/vklimkov/workspace/emp/NeMo/examples/tts/easymagpie_vllm_omni/easymp_vllm_model\")\n", + "assert (MODEL_DIR / \"config.json\").exists(), f\"No config.json under {MODEL_DIR}; run the converter first.\"\n", + "\n", + "config = json.loads((MODEL_DIR / \"config.json\").read_text())\n", + "arch = EasyMagpieOmniArch.from_hf_config(type(\"Cfg\", (), config))\n", + "\n", + "TEXT_VOCAB = int(config[\"text_vocab_size\"])\n", + "TEXT_EOS_ID = TEXT_VOCAB - 2\n", + "AUDIO_STOP_TOKEN_ID = max(1, int(config.get(\"vocab_size\", 2)) - 1)\n", + "SPEECH_DELAY = int(getattr(arch, \"streaming_speech_delay\", 0) or 0)\n", + "\n", + "print(f\"Model dir : {MODEL_DIR}\")\n", + "print(f\"num_stacked_codebooks : {arch.num_stacked_codebooks} (C*S)\")\n", + "print(f\"audio_bos / audio_eos id : {arch.audio_bos_id} / {arch.audio_eos_id}\")\n", + "print(f\"text_vocab / text_eos : {TEXT_VOCAB} / {TEXT_EOS_ID}\")\n", + "print(f\"audio-EOS stop token id : {AUDIO_STOP_TOKEN_ID}\")\n", + "print(f\"streaming speech delay : {SPEECH_DELAY} frames\")" + ] + }, + { + "cell_type": "markdown", + "id": "012df58d", + "metadata": {}, + "source": [ + "## 2. Single-stage `AsyncOmni` engine\n", + "\n", + "One `llm` stage running the EasyMagpie talker (`worker_type=\"ar\"`).\n", + "`final_output_type=\"audio\"` makes the runner attach the per-step `audio_codes`\n", + "payload to each output. `async_chunk=True` lets the streaming-text path (§5) feed\n", + "one subword per step and is a no-op for the whole-text path (§4), so a single\n", + "engine serves both." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5085e9a4", + "metadata": {}, + "outputs": [], + "source": [ + "DECODE_STEPS = 256 # max audio frames to decode (trimmed at audio EOS)\n", + "MAX_MODEL_LEN = 1024\n", + "MAX_NUM_BATCHED_TOKENS = 1024\n", + "\n", + "stage_cfg = {\n", + " \"async_chunk\": True,\n", + " \"stage_args\": [\n", + " {\n", + " \"stage_id\": 0,\n", + " \"stage_type\": \"llm\",\n", + " \"is_comprehension\": True,\n", + " \"final_output\": True,\n", + " \"final_output_type\": \"audio\",\n", + " \"runtime\": {\"devices\": \"0\"},\n", + " \"engine_args\": {\n", + " \"model_stage\": \"easymagpie\",\n", + " \"max_num_seqs\": 1,\n", + " \"model_arch\": \"EasyMagpieTTSForConditionalGeneration\",\n", + " \"worker_type\": \"ar\",\n", + " \"scheduler_cls\": \"easymagpie_vllm_omni.scheduler.EasyMagpieARAsyncScheduler\",\n", + " \"trust_remote_code\": True,\n", + " \"async_scheduling\": True,\n", + " \"enable_prefix_caching\": False,\n", + " \"enforce_eager\": False,\n", + " \"engine_output_type\": \"audio\",\n", + " \"gpu_memory_utilization\": 0.8,\n", + " \"distributed_executor_backend\": \"uni\",\n", + " \"max_num_batched_tokens\": MAX_NUM_BATCHED_TOKENS,\n", + " \"max_model_len\": MAX_MODEL_LEN,\n", + " \"dtype\": \"float16\",\n", + " \"mamba_ssm_cache_dtype\": \"float32\",\n", + " \"attention_backend\": \"TRITON_ATTN\",\n", + " \"skip_tokenizer_init\": True,\n", + " },\n", + " \"default_sampling_params\": {\n", + " \"temperature\": 0.0,\n", + " \"max_tokens\": DECODE_STEPS,\n", + " \"detokenize\": False,\n", + " \"ignore_eos\": True,\n", + " \"stop_token_ids\": [AUDIO_STOP_TOKEN_ID],\n", + " },\n", + " }\n", + " ],\n", + "}\n", + "\n", + "_tmp = tempfile.NamedTemporaryFile(\n", + " mode=\"w\", suffix=\".yaml\", prefix=\"easymagpie_omni_demo_\", delete=False,\n", + ")\n", + "yaml.dump(stage_cfg, _tmp, sort_keys=False)\n", + "_tmp.close()\n", + "STAGE_CFG_PATH = _tmp.name\n", + "print(f\"Stage config: {STAGE_CFG_PATH}\")\n", + "\n", + "omni = AsyncOmni(\n", + " model=str(MODEL_DIR),\n", + " stage_configs_path=STAGE_CFG_PATH,\n", + " log_stats=False,\n", + " stage_init_timeout=300,\n", + ")\n", + "print(\"Engine ready (single stage: EasyMagpie talker)\")" + ] + }, + { + "cell_type": "markdown", + "id": "2736b86d", + "metadata": {}, + "source": [ + "## 3. Build the prompt\n", + "\n", + "Per-request inputs passed via `additional_information`: `speaker_embedding`\n", + "`(T_audio, embedding_dim)`, the `context_text` string, and the target `text` (all\n", + "tokenized in-engine). `prompt_token_ids` are placeholders sized by\n", + "`estimate_prompt_len(...)` to match the assembled prefill context. Audio sampling\n", + "`temperature` / `top_k` are forwarded to the local transformer." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "697c74b3", + "metadata": {}, + "outputs": [], + "source": [ + "torch.manual_seed(0)\n", + "\n", + "from transformers import AutoTokenizer\n", + "\n", + "from easymagpie_vllm_omni.easymagpie import EasyMagpieTTSForConditionalGeneration\n", + "\n", + "SPEAKER_NAME = \"eng\"\n", + "_loaded = torch.load(MODEL_DIR / \"speaker_embeddings\" / f\"{SPEAKER_NAME}.pt\", map_location=\"cpu\")\n", + "speaker_embedding = _loaded[\"speaker_encoding\"] if isinstance(_loaded, dict) else _loaded\n", + "speaker_embedding = speaker_embedding.to(torch.float32)\n", + "\n", + "CONTEXT_TEXT = \"[EN]\"\n", + "TEXT = \"Hello, this is a test of the EasyMagpie text to speech model.\"\n", + "\n", + "# Audio sampling (local transformer); temperature=0 == argmax (deterministic).\n", + "LT_TEMPERATURE = 0.0\n", + "LT_TOPK = 80\n", + "\n", + "# Same tokenizer the engine loads from MODEL_DIR; used only to size the placeholders.\n", + "tokenizer = AutoTokenizer.from_pretrained(MODEL_DIR, trust_remote_code=True)\n", + "prompt_len = EasyMagpieTTSForConditionalGeneration.estimate_prompt_len(\n", + " speaker_embedding,\n", + " tokenize=lambda t: tokenizer.encode(t),\n", + " context_text=CONTEXT_TEXT,\n", + " has_task_embedding=arch.num_task_embeddings > 0,\n", + ")\n", + "\n", + "prompt = {\n", + " \"prompt_token_ids\": [0] * prompt_len,\n", + " \"additional_information\": {\n", + " \"speaker_embedding\": speaker_embedding,\n", + " \"context_text\": CONTEXT_TEXT,\n", + " \"text\": TEXT,\n", + " \"temperature\": LT_TEMPERATURE,\n", + " \"top_k\": LT_TOPK,\n", + " },\n", + "}\n", + "assert prompt_len + DECODE_STEPS <= MAX_MODEL_LEN\n", + "\n", + "# output_kind=DELTA: audio_codes arrives as a growing list during decode\n", + "# ([prefill_prefix, frame_0, frame_1, ...]); prefill/final steps yield a tensor.\n", + "sampling_params = SamplingParams(\n", + " temperature=0.0,\n", + " max_tokens=DECODE_STEPS,\n", + " detokenize=False,\n", + " ignore_eos=True,\n", + " stop_token_ids=[AUDIO_STOP_TOKEN_ID],\n", + " output_kind=RequestOutputKind.DELTA,\n", + ")\n", + "\n", + "print(f\"speaker_embedding : {tuple(speaker_embedding.shape)}\")\n", + "print(f\"context_text / text : {CONTEXT_TEXT!r} / {TEXT!r}\")\n", + "print(f\"prompt_len (placeholders) : {prompt_len}\")" + ] + }, + { + "cell_type": "markdown", + "id": "3ef8934d", + "metadata": {}, + "source": [ + "## 4. Whole-text inference -> audio codes\n", + "\n", + "`omni.generate(...)` yields one `RequestOutput` per engine step, each carrying the\n", + "**cumulative** `multimodal_output[\"audio_codes\"]` tensor: the first `prompt_len`\n", + "rows are the prefill prefix, followed by one `(1, C*S)` decoded frame per step\n", + "(occasionally surfaced as a list of per-step tensors, which we just concatenate).\n", + "We keep the largest tensor seen, drop the prefill prefix, then drop the\n", + "`speech_delay` warm-up frames and the trailing audio-EOS frame." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6d0ccbd4", + "metadata": {}, + "outputs": [], + "source": [ + "async def run_request(prompt, sampling_params):\n", + " \"\"\"Keep the cumulative audio-code tensor [prefill_prefix | decoded frames].\n", + "\n", + " Each step exposes audio_codes as a growing tensor (sometimes a list of per-step\n", + " tensors, which we concatenate); we keep the largest one seen.\n", + " \"\"\"\n", + " request_id = f\"easymagpie-{uuid.uuid4().hex[:8]}\"\n", + " codes = None\n", + " async for out in omni.generate(prompt, sampling_params_list=[sampling_params], request_id=request_id):\n", + " payload = (out.multimodal_output or {}).get(\"audio_codes\")\n", + " if isinstance(payload, list):\n", + " payload = torch.cat([t for t in payload if isinstance(t, torch.Tensor)], dim=0) if payload else None\n", + " if isinstance(payload, torch.Tensor) and (codes is None or payload.shape[0] >= codes.shape[0]):\n", + " codes = payload.detach().cpu().to(torch.long)\n", + " return codes\n", + "\n", + "\n", + "codes = await run_request(prompt, sampling_params)\n", + "# Rows 0:prompt_len are the prefill prefix; decoded frames follow. Drop the prefix,\n", + "# the speech-delay warm-up frames, and the trailing audio-EOS frame.\n", + "audio_codes = codes[prompt_len + SPEECH_DELAY : -1]\n", + "print(f\"cumulative codes : {tuple(codes.shape)} (prompt_len={prompt_len})\")\n", + "print(f\"audio_codes : {tuple(audio_codes.shape)} (dropped prefix + {SPEECH_DELAY} warm-up + 1 EOS)\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "04196662", + "metadata": {}, + "outputs": [], + "source": [ + "import matplotlib.pylab as plt\n", + "\n", + "plt.figure(figsize=(10, 3))\n", + "plt.imshow(audio_codes.T, aspect=\"auto\", interpolation=\"nearest\")\n", + "plt.title(f\"whole-text audio codes ({audio_codes.shape[0]} frames)\")\n", + "plt.xlabel(\"decode frame\")\n", + "plt.ylabel(\"stacked codebook\")\n", + "plt.colorbar()\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "a32b07d5", + "metadata": {}, + "source": [ + "## 5. Streaming text input (one subword per decode step)\n", + "\n", + "Same model, but push subword ids **one at a time** as it decodes (the live-client\n", + "path). `omni.generate(...)` is given an async generator of `StreamingInput` chunks:\n", + "chunk 0 is the prefill (speaker + context, no text), each later chunk carries one\n", + "`text_token` with `max_tokens=1` (one chunk -> one decoded frame). After the text\n", + "is exhausted we feed the `-1` mask sentinel until the model emits the audio-EOS\n", + "token. Input is paced by output via `go_queue`. Codes are collected exactly as in\n", + "§4 (keep the cumulative tensor, slice off prefix + warm-up + EOS)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "aa57a573", + "metadata": {}, + "outputs": [], + "source": [ + "import asyncio\n", + "from collections.abc import AsyncGenerator\n", + "\n", + "try:\n", + " from vllm.engine.protocol import StreamingInput\n", + "except ImportError:\n", + " from vllm.v1.engine.async_llm import StreamingInput\n", + "\n", + "# Tokenize the target text the same way the model does (specials off + text-EOS).\n", + "text_ids = list(tokenizer.encode(TEXT, add_special_tokens=False)) + [TEXT_EOS_ID]\n", + "print(f\"streamed text ids ({len(text_ids)}): {text_ids}\")\n", + "\n", + "stream_params = SamplingParams(\n", + " temperature=0.0,\n", + " max_tokens=1,\n", + " detokenize=False,\n", + " ignore_eos=True,\n", + " stop_token_ids=[AUDIO_STOP_TOKEN_ID],\n", + " output_kind=RequestOutputKind.DELTA,\n", + ")\n", + "\n", + "go_queue: asyncio.Queue[bool] = asyncio.Queue()\n", + "\n", + "\n", + "async def stream_text_inputs() -> AsyncGenerator[StreamingInput, None]:\n", + " # Prefill: speaker + context, NO text (its absence selects streaming-text mode).\n", + " # The first subword rides on the first decode chunk, not the prefill.\n", + " prefill_info = {\n", + " \"speaker_embedding\": speaker_embedding,\n", + " \"context_text\": CONTEXT_TEXT,\n", + " \"temperature\": LT_TEMPERATURE,\n", + " \"top_k\": LT_TOPK,\n", + " }\n", + " yield StreamingInput(\n", + " prompt={\"prompt_token_ids\": [0] * prompt_len, \"additional_information\": prefill_info},\n", + " sampling_params=stream_params,\n", + " )\n", + " # One chunk per decode frame, paced by go_queue: text_ids[k] on decode step k,\n", + " # then the -1 mask once the text is exhausted.\n", + " step = 0\n", + " while True:\n", + " await go_queue.get()\n", + " tok = int(text_ids[step]) if step < len(text_ids) else -1\n", + " yield StreamingInput(\n", + " prompt={\"prompt_token_ids\": [0], \"additional_information\": {\"text_token\": tok}},\n", + " sampling_params=stream_params,\n", + " )\n", + " step += 1\n", + "\n", + "\n", + "async def run_streaming_request():\n", + " \"\"\"Collect codes like §4, pacing one chunk per decoded frame.\n", + "\n", + " Every streaming segment reports finish_reason \"length\"; the request truly ends\n", + " only at the audio-EOS stop token, where we stop pacing and break.\n", + " \"\"\"\n", + " request_id = f\"easymagpie-stream-{uuid.uuid4().hex[:8]}\"\n", + " codes = None\n", + " MAX_STEPS = 4 * DECODE_STEPS + 16\n", + " steps = 0\n", + " async for out in omni.generate(\n", + " stream_text_inputs(), sampling_params_list=[stream_params], request_id=request_id\n", + " ):\n", + " steps += 1\n", + " co = out.outputs[0] if out.outputs else None\n", + " payload = (out.multimodal_output or {}).get(\"audio_codes\")\n", + " if isinstance(payload, list):\n", + " payload = torch.cat([t for t in payload if isinstance(t, torch.Tensor)], dim=0) if payload else None\n", + " if isinstance(payload, torch.Tensor) and (codes is None or payload.shape[0] >= codes.shape[0]):\n", + " codes = payload.detach().cpu().to(torch.long)\n", + " if getattr(co, \"stop_reason\", None) == AUDIO_STOP_TOKEN_ID or steps >= MAX_STEPS:\n", + " break\n", + " await go_queue.put(True)\n", + " return codes\n", + "\n", + "\n", + "codes_stream = await run_streaming_request()\n", + "audio_codes_stream = codes_stream[prompt_len + SPEECH_DELAY : -1]\n", + "print(f\"cumulative codes : {tuple(codes_stream.shape)}\")\n", + "print(f\"audio_codes : {tuple(audio_codes_stream.shape)}\")" + ] + }, + { + "cell_type": "markdown", + "id": "4d86ebf5", + "metadata": {}, + "source": [ + "## 6. Decode codes to waveforms\n", + "\n", + "The engine emits **stacked** codes `(T, C*S)` with `C*S = 16`. To vocode we mirror\n", + "`EasyMagpieTTSInferenceModel`: load the `.nemo` codec, unstack `(T, C*S)` ->\n", + "`(1, C, T*S)`, optionally remap the regrouped FSQ token space back to the codec's\n", + "native space, then `codec_model.decode(...)`.\n", + "\n", + "Set `CODEC_MODEL_PATH` / `EASYMAGPIE_NEMO` to the same `.nemo` files passed to the\n", + "converter. Needs NeMo importable in this environment." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3ab9fedc", + "metadata": {}, + "outputs": [], + "source": [ + "from hydra.utils import instantiate\n", + "from IPython.display import Audio, display\n", + "\n", + "from nemo.collections.tts.models import AudioCodecModel\n", + "from nemo.collections.tts.models.easy_magpietts_inference import EasyMagpieTTSInferenceModel\n", + "from nemo.collections.tts.modules.audio_codec_modules import VectorQuantizerIndexConverter\n", + "\n", + "CODEC_MODEL_PATH = \"/home/vklimkov/workspace/emp/ckpt/easymagpietts_NEXT/25fps_spectral_codec_with_bandwidth_extension.nemo\"\n", + "EASYMAGPIE_NEMO = \"/home/vklimkov/workspace/emp/ckpt/easymagpietts_NEXT/2605_NemotronTTS_V0.2/v2/2605_EMTTS_SmallMamba_Step150k_posttrained_epoch12.nemo\"\n", + "\n", + "# Load the codec once (drop the discriminator to save memory).\n", + "_codec_cfg = AudioCodecModel.restore_from(CODEC_MODEL_PATH, return_config=True)\n", + "if \"use_scl_loss\" in _codec_cfg:\n", + " _codec_cfg.use_scl_loss = False\n", + "codec_model = AudioCodecModel.restore_from(CODEC_MODEL_PATH, strict=False, override_config_path=_codec_cfg)\n", + "if hasattr(codec_model, \"discriminator\"):\n", + " del codec_model.discriminator\n", + "codec_model = codec_model.eval().to(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", + "codec_device = next(codec_model.parameters()).device\n", + "\n", + "# The model may use a regrouped FSQ token space; map it back to the codec's native\n", + "# space when they differ (config read from the source EasyMagpie .nemo, no weights).\n", + "_em_cfg = EasyMagpieTTSInferenceModel.restore_from(EASYMAGPIE_NEMO, return_config=True)\n", + "_vq_cfg = _em_cfg.get(\"vector_quantizer\")\n", + "if _vq_cfg is not None and instantiate(_vq_cfg).num_codebooks != codec_model.vector_quantizer.num_codebooks:\n", + " codec_converter = VectorQuantizerIndexConverter(\n", + " vector_quantizer_original=codec_model.vector_quantizer,\n", + " vector_quantizer_new=instantiate(_vq_cfg),\n", + " ).to(codec_device)\n", + "else:\n", + " codec_converter = None\n", + "print(f\"codec native codebooks : {codec_model.vector_quantizer.num_codebooks}\")\n", + "print(f\"codec token converter : {'enabled' if codec_converter is not None else 'not needed'}\")\n", + "\n", + "S = arch.frame_stacking_factor\n", + "C = arch.num_stacked_codebooks // S\n", + "sample_rate = int(codec_model.output_sample_rate)\n", + "\n", + "\n", + "def decode_codes_to_waveform(audio_codes: torch.Tensor):\n", + " \"\"\"Decode one (T, C*S) stacked-code sequence to a mono float32 waveform.\"\"\"\n", + " stacked = audio_codes.to(codec_device, torch.long).T.unsqueeze(0) # (1, C*S, T)\n", + " T_out = stacked.size(-1)\n", + " codes = stacked.view(1, C, S, T_out).permute(0, 1, 3, 2).reshape(1, C, T_out * S) # (1, C, T*S)\n", + " codes_len = torch.tensor([codes.size(-1)], device=codec_device, dtype=torch.long)\n", + "\n", + " MIN_LEN = 4\n", + " if int(codes_len.min()) < MIN_LEN:\n", + " codes = torch.nn.functional.pad(codes, (0, MIN_LEN - int(codes_len.min())), value=0)\n", + " codes_len = codes_len.clamp(min=MIN_LEN)\n", + " codes = codes.clamp_(0, arch.codebook_size - 1)\n", + "\n", + " with torch.no_grad(), torch.autocast(device_type=codec_device.type, dtype=torch.float32):\n", + " if codec_converter is not None:\n", + " codes = codec_converter.convert_new_to_original(audio_tokens=codes, audio_lens=codes_len)\n", + " audio, audio_len = codec_model.decode(tokens=codes, tokens_len=codes_len)\n", + " return audio[0, : int(audio_len[0])].detach().cpu().float().numpy()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0e6fad73", + "metadata": {}, + "outputs": [], + "source": [ + "# Vocode and play each run.\n", + "for name, codes in [(\"whole-text (§4)\", audio_codes), (\"streamed text (§5)\", audio_codes_stream)]:\n", + " wav = decode_codes_to_waveform(codes)\n", + " print(f\"{name:<20}: {tuple(codes.shape)} codes -> {wav.shape[0] / sample_rate:.2f}s @ {sample_rate} Hz\")\n", + " display(Audio(wav, rate=sample_rate))" + ] + }, + { + "cell_type": "markdown", + "id": "d670e328", + "metadata": {}, + "source": [ + "### Streamed-text audio codes" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "77b12390", + "metadata": {}, + "outputs": [], + "source": [ + "import matplotlib.pylab as plt\n", + "\n", + "plt.figure(figsize=(10, 3))\n", + "plt.imshow(audio_codes_stream.T, aspect=\"auto\", interpolation=\"nearest\")\n", + "plt.title(f\"streamed-text audio codes ({audio_codes_stream.shape[0]} frames)\")\n", + "plt.xlabel(\"decode frame\")\n", + "plt.ylabel(\"stacked codebook\")\n", + "plt.colorbar()\n", + "plt.show()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "emp", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.13" + } + }, + "nbformat": 4, + "nbformat_minor": 5 } From bc51d8d678fbcc31031072a39c5a2373e4e0a0ec Mon Sep 17 00:00:00 2001 From: Viacheslav Klimkov Date: Wed, 10 Jun 2026 00:34:24 +0200 Subject: [PATCH 30/45] examples/tts/easymagpie_vllm_omni: custom scheduler to resume after text tokens are all streamed Signed-off-by: Viacheslav Klimkov --- .../easymagpie_inference_demo.ipynb | 1126 +++++++++-------- .../easymagpie_vllm_omni/scheduler.py | 75 ++ 2 files changed, 646 insertions(+), 555 deletions(-) create mode 100644 examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/scheduler.py diff --git a/examples/tts/easymagpie_vllm_omni/easymagpie_inference_demo.ipynb b/examples/tts/easymagpie_vllm_omni/easymagpie_inference_demo.ipynb index 65615ea2d2ee..6f4b746aa655 100644 --- a/examples/tts/easymagpie_vllm_omni/easymagpie_inference_demo.ipynb +++ b/examples/tts/easymagpie_vllm_omni/easymagpie_inference_demo.ipynb @@ -1,558 +1,574 @@ { - "cells": [ - { - "cell_type": "markdown", - "id": "d5a1129d", - "metadata": {}, - "source": [ - "# EasyMagpieTTS — vLLM-Omni inference demo\n", - "\n", - "End-to-end inference through `easymagpie_vllm_omni` on a checkpoint converted by\n", - "`easy_magpietts_convert_to_vllm.py`: prefill (speaker embedding + context text) ->\n", - "autoregressive decode (target text, one subword per step) -> stacked audio codes ->\n", - "vocode to waveform.\n", - "\n", - "Run inside the `vllm_omni_env` with the plugin installed\n", - "(`pip install -e examples/tts/easymagpie_vllm_omni`)." - ] + "cells": [ + { + "cell_type": "markdown", + "id": "d5a1129d", + "metadata": {}, + "source": [ + "# EasyMagpieTTS — vLLM-Omni inference demo\n", + "\n", + "End-to-end inference through `easymagpie_vllm_omni` on a checkpoint converted by\n", + "`easy_magpietts_convert_to_vllm.py`: prefill (speaker embedding + context text) ->\n", + "autoregressive decode (target text, one subword per step) -> stacked audio codes ->\n", + "vocode to waveform.\n", + "\n", + "Run inside the `vllm_omni_env` with the plugin installed\n", + "(`pip install -e examples/tts/easymagpie_vllm_omni`)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c9a71b74", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "os.environ.setdefault(\"VLLM_WORKER_MULTIPROC_METHOD\", \"spawn\")\n", + "\n", + "import json\n", + "import tempfile\n", + "import uuid\n", + "from pathlib import Path\n", + "\n", + "import torch\n", + "import yaml\n", + "\n", + "from vllm import SamplingParams\n", + "from vllm.sampling_params import RequestOutputKind\n", + "from vllm_omni import AsyncOmni\n", + "\n", + "from easymagpie_vllm_omni.config import EasyMagpieOmniArch\n", + "\n", + "print(\"torch:\", torch.__version__, \"| cuda:\", torch.cuda.is_available())" + ] + }, + { + "cell_type": "markdown", + "id": "f7ff55fe", + "metadata": {}, + "source": [ + "## 1. Converted model directory\n", + "\n", + "`MODEL_DIR` is the directory written by the converter: `config.json`, weights, the\n", + "text tokenizer, and per-speaker embeddings. The engine loads everything from it;\n", + "here we only read a few config scalars used to build the prompt." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3e0df89e", + "metadata": {}, + "outputs": [], + "source": [ + "MODEL_DIR = Path(\"/home/vklimkov/workspace/emp/NeMo/examples/tts/easymagpie_vllm_omni/easymp_vllm_model\")\n", + "assert (MODEL_DIR / \"config.json\").exists(), f\"No config.json under {MODEL_DIR}; run the converter first.\"\n", + "\n", + "config = json.loads((MODEL_DIR / \"config.json\").read_text())\n", + "arch = EasyMagpieOmniArch.from_hf_config(type(\"Cfg\", (), config))\n", + "\n", + "TEXT_VOCAB = int(config[\"text_vocab_size\"])\n", + "TEXT_EOS_ID = TEXT_VOCAB - 2\n", + "AUDIO_STOP_TOKEN_ID = max(1, int(config.get(\"vocab_size\", 2)) - 1)\n", + "SPEECH_DELAY = int(getattr(arch, \"streaming_speech_delay\", 0) or 0)\n", + "\n", + "print(f\"Model dir : {MODEL_DIR}\")\n", + "print(f\"num_stacked_codebooks : {arch.num_stacked_codebooks} (C*S)\")\n", + "print(f\"audio_bos / audio_eos id : {arch.audio_bos_id} / {arch.audio_eos_id}\")\n", + "print(f\"text_vocab / text_eos : {TEXT_VOCAB} / {TEXT_EOS_ID}\")\n", + "print(f\"audio-EOS stop token id : {AUDIO_STOP_TOKEN_ID}\")\n", + "print(f\"streaming speech delay : {SPEECH_DELAY} frames\")" + ] + }, + { + "cell_type": "markdown", + "id": "012df58d", + "metadata": {}, + "source": [ + "## 2. Single-stage `AsyncOmni` engine\n", + "\n", + "One `llm` stage running the EasyMagpie talker (`worker_type=\"ar\"`).\n", + "`final_output_type=\"audio\"` makes the runner attach the per-step `audio_codes`\n", + "payload to each output. `async_chunk=True` lets the streaming-text path (§5) feed\n", + "one subword per step and is a no-op for the whole-text path (§4), so a single\n", + "engine serves both." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5085e9a4", + "metadata": {}, + "outputs": [], + "source": [ + "DECODE_STEPS = 256 # max audio frames to decode (trimmed at audio EOS)\n", + "MAX_MODEL_LEN = 1024\n", + "MAX_NUM_BATCHED_TOKENS = 1024\n", + "\n", + "stage_cfg = {\n", + " \"async_chunk\": True,\n", + " \"stage_args\": [\n", + " {\n", + " \"stage_id\": 0,\n", + " \"stage_type\": \"llm\",\n", + " \"is_comprehension\": True,\n", + " \"final_output\": True,\n", + " \"final_output_type\": \"audio\",\n", + " \"runtime\": {\"devices\": \"0\"},\n", + " \"engine_args\": {\n", + " \"model_stage\": \"easymagpie\",\n", + " \"max_num_seqs\": 1,\n", + " \"model_arch\": \"EasyMagpieTTSForConditionalGeneration\",\n", + " \"worker_type\": \"ar\",\n", + " \"scheduler_cls\": \"easymagpie_vllm_omni.scheduler.EasyMagpieARAsyncScheduler\",\n", + " \"trust_remote_code\": True,\n", + " \"async_scheduling\": True,\n", + " \"enable_prefix_caching\": False,\n", + " \"enforce_eager\": False,\n", + " \"engine_output_type\": \"audio\",\n", + " \"gpu_memory_utilization\": 0.8,\n", + " \"distributed_executor_backend\": \"uni\",\n", + " \"max_num_batched_tokens\": MAX_NUM_BATCHED_TOKENS,\n", + " \"max_model_len\": MAX_MODEL_LEN,\n", + " \"dtype\": \"float16\",\n", + " \"mamba_ssm_cache_dtype\": \"float32\",\n", + " \"attention_backend\": \"TRITON_ATTN\",\n", + " \"skip_tokenizer_init\": True,\n", + " },\n", + " \"default_sampling_params\": {\n", + " \"temperature\": 0.0,\n", + " \"max_tokens\": DECODE_STEPS,\n", + " \"detokenize\": False,\n", + " \"ignore_eos\": True,\n", + " \"stop_token_ids\": [AUDIO_STOP_TOKEN_ID],\n", + " },\n", + " }\n", + " ],\n", + "}\n", + "\n", + "_tmp = tempfile.NamedTemporaryFile(\n", + " mode=\"w\", suffix=\".yaml\", prefix=\"easymagpie_omni_demo_\", delete=False,\n", + ")\n", + "yaml.dump(stage_cfg, _tmp, sort_keys=False)\n", + "_tmp.close()\n", + "STAGE_CFG_PATH = _tmp.name\n", + "print(f\"Stage config: {STAGE_CFG_PATH}\")\n", + "\n", + "omni = AsyncOmni(\n", + " model=str(MODEL_DIR),\n", + " stage_configs_path=STAGE_CFG_PATH,\n", + " log_stats=False,\n", + " stage_init_timeout=300,\n", + ")\n", + "print(\"Engine ready (single stage: EasyMagpie talker)\")" + ] + }, + { + "cell_type": "markdown", + "id": "2736b86d", + "metadata": {}, + "source": [ + "## 3. Build the prompt\n", + "\n", + "Per-request inputs passed via `additional_information`: `speaker_embedding`\n", + "`(T_audio, embedding_dim)`, the `context_text` string, and the target `text` (all\n", + "tokenized in-engine). `prompt_token_ids` are placeholders sized by\n", + "`estimate_prompt_len(...)` to match the assembled prefill context. Audio sampling\n", + "`temperature` / `top_k` are forwarded to the local transformer." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "697c74b3", + "metadata": {}, + "outputs": [], + "source": [ + "torch.manual_seed(0)\n", + "\n", + "from transformers import AutoTokenizer\n", + "\n", + "from easymagpie_vllm_omni.easymagpie import EasyMagpieTTSForConditionalGeneration\n", + "\n", + "SPEAKER_NAME = \"eng\"\n", + "_loaded = torch.load(MODEL_DIR / \"speaker_embeddings\" / f\"{SPEAKER_NAME}.pt\", map_location=\"cpu\")\n", + "speaker_embedding = _loaded[\"speaker_encoding\"] if isinstance(_loaded, dict) else _loaded\n", + "speaker_embedding = speaker_embedding.to(torch.float32)\n", + "\n", + "CONTEXT_TEXT = \"[EN]\"\n", + "TEXT = \"Hello, this is a test of the EasyMagpie text to speech model.\"\n", + "\n", + "# Audio sampling (local transformer); temperature=0 == argmax (deterministic).\n", + "LT_TEMPERATURE = 0.0\n", + "LT_TOPK = 80\n", + "\n", + "# Same tokenizer the engine loads from MODEL_DIR; used only to size the placeholders.\n", + "tokenizer = AutoTokenizer.from_pretrained(MODEL_DIR, trust_remote_code=True)\n", + "prompt_len = EasyMagpieTTSForConditionalGeneration.estimate_prompt_len(\n", + " speaker_embedding,\n", + " tokenize=lambda t: tokenizer.encode(t),\n", + " context_text=CONTEXT_TEXT,\n", + " has_task_embedding=arch.num_task_embeddings > 0,\n", + ")\n", + "\n", + "prompt = {\n", + " \"prompt_token_ids\": [0] * prompt_len,\n", + " \"additional_information\": {\n", + " \"speaker_embedding\": speaker_embedding,\n", + " \"context_text\": CONTEXT_TEXT,\n", + " \"text\": TEXT,\n", + " \"temperature\": LT_TEMPERATURE,\n", + " \"top_k\": LT_TOPK,\n", + " },\n", + "}\n", + "assert prompt_len + DECODE_STEPS <= MAX_MODEL_LEN\n", + "\n", + "# output_kind=DELTA: audio_codes arrives as a growing list during decode\n", + "# ([prefill_prefix, frame_0, frame_1, ...]); prefill/final steps yield a tensor.\n", + "sampling_params = SamplingParams(\n", + " temperature=0.0,\n", + " max_tokens=DECODE_STEPS,\n", + " detokenize=False,\n", + " ignore_eos=True,\n", + " stop_token_ids=[AUDIO_STOP_TOKEN_ID],\n", + " output_kind=RequestOutputKind.DELTA,\n", + ")\n", + "\n", + "print(f\"speaker_embedding : {tuple(speaker_embedding.shape)}\")\n", + "print(f\"context_text / text : {CONTEXT_TEXT!r} / {TEXT!r}\")\n", + "print(f\"prompt_len (placeholders) : {prompt_len}\")" + ] + }, + { + "cell_type": "markdown", + "id": "3ef8934d", + "metadata": {}, + "source": [ + "## 4. Whole-text inference -> audio codes\n", + "\n", + "`omni.generate(...)` yields one `RequestOutput` per engine step, each carrying the\n", + "**cumulative** `multimodal_output[\"audio_codes\"]` tensor: the first `prompt_len`\n", + "rows are the prefill prefix, followed by one `(1, C*S)` decoded frame per step\n", + "(occasionally surfaced as a list of per-step tensors, which we just concatenate).\n", + "We keep the largest tensor seen, drop the prefill prefix, then drop the\n", + "`speech_delay` warm-up frames and the trailing audio-EOS frame." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6d0ccbd4", + "metadata": {}, + "outputs": [], + "source": [ + "async def run_request(prompt, sampling_params):\n", + " \"\"\"Keep the cumulative audio-code tensor [prefill_prefix | decoded frames].\n", + "\n", + " Each step exposes audio_codes as a growing tensor (sometimes a list of per-step\n", + " tensors, which we concatenate); we keep the largest one seen.\n", + " \"\"\"\n", + " request_id = f\"easymagpie-{uuid.uuid4().hex[:8]}\"\n", + " codes = None\n", + " async for out in omni.generate(prompt, sampling_params_list=[sampling_params], request_id=request_id):\n", + " payload = (out.multimodal_output or {}).get(\"audio_codes\")\n", + " if isinstance(payload, list):\n", + " payload = torch.cat([t for t in payload if isinstance(t, torch.Tensor)], dim=0) if payload else None\n", + " if isinstance(payload, torch.Tensor) and (codes is None or payload.shape[0] >= codes.shape[0]):\n", + " codes = payload.detach().cpu().to(torch.long)\n", + " return codes\n", + "\n", + "\n", + "codes = await run_request(prompt, sampling_params)\n", + "# Rows 0:prompt_len are the prefill prefix; decoded frames follow. Drop the prefix,\n", + "# the speech-delay warm-up frames, and the trailing audio-EOS frame.\n", + "audio_codes = codes[prompt_len + SPEECH_DELAY : -1]\n", + "print(f\"cumulative codes : {tuple(codes.shape)} (prompt_len={prompt_len})\")\n", + "print(f\"audio_codes : {tuple(audio_codes.shape)} (dropped prefix + {SPEECH_DELAY} warm-up + 1 EOS)\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "04196662", + "metadata": {}, + "outputs": [], + "source": [ + "import matplotlib.pylab as plt\n", + "\n", + "plt.figure(figsize=(10, 3))\n", + "plt.imshow(audio_codes.T, aspect=\"auto\", interpolation=\"nearest\")\n", + "plt.title(f\"whole-text audio codes ({audio_codes.shape[0]} frames)\")\n", + "plt.xlabel(\"decode frame\")\n", + "plt.ylabel(\"stacked codebook\")\n", + "plt.colorbar()\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "a32b07d5", + "metadata": {}, + "source": [ + "## 5. Streaming text input (one subword per decode step)\n", + "\n", + "Same model, but push subword ids **one at a time** as it decodes (the live-client\n", + "path). `omni.generate(...)` is given an async generator of `StreamingInput` chunks:\n", + "chunk 0 is the prefill (speaker + context, no text), each later chunk carries one\n", + "`text_token` with `max_tokens=1` (one chunk -> one decoded frame). After the text\n", + "is exhausted we feed the `-1` mask sentinel until the model emits the audio-EOS\n", + "token. Input is paced by output via `go_queue`. Codes are collected exactly as in\n", + "§4 (keep the cumulative tensor, slice off prefix + warm-up + EOS)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "aa57a573", + "metadata": {}, + "outputs": [], + "source": [ + "import asyncio\n", + "from collections.abc import AsyncGenerator\n", + "\n", + "try:\n", + " from vllm.engine.protocol import StreamingInput\n", + "except ImportError:\n", + " from vllm.v1.engine.async_llm import StreamingInput\n", + "\n", + "# Tokenize the target text the same way the model does (specials off + text-EOS).\n", + "text_ids = list(tokenizer.encode(TEXT, add_special_tokens=False)) + [TEXT_EOS_ID]\n", + "print(f\"streamed text ids ({len(text_ids)}): {text_ids}\")\n", + "\n", + "stream_params = SamplingParams(\n", + " temperature=0.0,\n", + " max_tokens=1,\n", + " detokenize=False,\n", + " ignore_eos=True,\n", + " stop_token_ids=[AUDIO_STOP_TOKEN_ID],\n", + " output_kind=RequestOutputKind.DELTA,\n", + ")\n", + "TAIL_MAX_TOKENS = DECODE_STEPS - len(text_ids)\n", + "tail_params = SamplingParams(\n", + " temperature=0.0,\n", + " max_tokens=TAIL_MAX_TOKENS,\n", + " detokenize=False,\n", + " ignore_eos=True,\n", + " stop_token_ids=[AUDIO_STOP_TOKEN_ID],\n", + " output_kind=RequestOutputKind.DELTA,\n", + ")\n", + "\n", + "go_queue: asyncio.Queue[bool] = asyncio.Queue()\n", + "\n", + "\n", + "async def stream_text_inputs() -> AsyncGenerator[StreamingInput, None]:\n", + " # Prefill: speaker + context, NO text (its absence selects streaming-text mode).\n", + " # The first subword rides on the first decode chunk, not the prefill.\n", + " prefill_info = {\n", + " \"speaker_embedding\": speaker_embedding,\n", + " \"context_text\": CONTEXT_TEXT,\n", + " \"temperature\": LT_TEMPERATURE,\n", + " \"top_k\": LT_TOPK,\n", + " }\n", + " yield StreamingInput(\n", + " prompt={\"prompt_token_ids\": [0] * prompt_len, \"additional_information\": prefill_info},\n", + " sampling_params=stream_params,\n", + " )\n", + "\n", + " for tok in text_ids:\n", + " await go_queue.get()\n", + " yield StreamingInput(\n", + " prompt={\"prompt_token_ids\": [0], \"additional_information\": {\"text_token\": int(tok)}},\n", + " sampling_params=stream_params,\n", + " )\n", + "\n", + " await go_queue.get()\n", + " yield StreamingInput(\n", + " prompt={\"prompt_token_ids\": [0], \"additional_information\": {\"text_token\": -1}},\n", + " sampling_params=tail_params,\n", + " )\n", + " # Generator ends here: the engine now decodes the acoustic tail uninterrupted\n", + " # on `tail_params.max_tokens` (honored thanks to the max_tokens propagation in\n", + " # EasyMagpieARAsyncScheduler), stopping at the audio-EOS token. Returning lets\n", + " # vLLM-Omni append a non-resumable finish sentinel that closes the session as\n", + " # a safety net if the tail ever exhausts max_tokens without emitting EOS.\n", + "\n", + "\n", + "async def run_streaming_request():\n", + " \"\"\"Collect codes like §4, pacing one chunk per decoded frame.\n", + "\n", + " Every streaming segment reports finish_reason \"length\"; the request truly ends\n", + " only at the audio-EOS stop token, where we stop pacing and break.\n", + " \"\"\"\n", + " request_id = f\"easymagpie-stream-{uuid.uuid4().hex[:8]}\"\n", + " codes = None\n", + " MAX_STEPS = 4 * DECODE_STEPS + 16\n", + " steps = 0\n", + " async for out in omni.generate(\n", + " stream_text_inputs(), sampling_params_list=[stream_params], request_id=request_id\n", + " ):\n", + " steps += 1\n", + " co = out.outputs[0] if out.outputs else None\n", + " payload = (out.multimodal_output or {}).get(\"audio_codes\")\n", + " if isinstance(payload, list):\n", + " payload = torch.cat([t for t in payload if isinstance(t, torch.Tensor)], dim=0) if payload else None\n", + " if isinstance(payload, torch.Tensor) and (codes is None or payload.shape[0] >= codes.shape[0]):\n", + " codes = payload.detach().cpu().to(torch.long)\n", + " if getattr(co, \"stop_reason\", None) == AUDIO_STOP_TOKEN_ID or steps >= MAX_STEPS:\n", + " break\n", + " await go_queue.put(True)\n", + " return codes\n", + "\n", + "\n", + "codes_stream = await run_streaming_request()\n", + "audio_codes_stream = codes_stream[prompt_len + SPEECH_DELAY : -1]\n", + "print(f\"cumulative codes : {tuple(codes_stream.shape)}\")\n", + "print(f\"audio_codes : {tuple(audio_codes_stream.shape)}\")" + ] + }, + { + "cell_type": "markdown", + "id": "4d86ebf5", + "metadata": {}, + "source": [ + "## 6. Decode codes to waveforms\n", + "\n", + "The engine emits **stacked** codes `(T, C*S)` with `C*S = 16`. To vocode we mirror\n", + "`EasyMagpieTTSInferenceModel`: load the `.nemo` codec, unstack `(T, C*S)` ->\n", + "`(1, C, T*S)`, optionally remap the regrouped FSQ token space back to the codec's\n", + "native space, then `codec_model.decode(...)`.\n", + "\n", + "Set `CODEC_MODEL_PATH` / `EASYMAGPIE_NEMO` to the same `.nemo` files passed to the\n", + "converter. Needs NeMo importable in this environment." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3ab9fedc", + "metadata": {}, + "outputs": [], + "source": [ + "from hydra.utils import instantiate\n", + "from IPython.display import Audio, display\n", + "\n", + "from nemo.collections.tts.models import AudioCodecModel\n", + "from nemo.collections.tts.models.easy_magpietts_inference import EasyMagpieTTSInferenceModel\n", + "from nemo.collections.tts.modules.audio_codec_modules import VectorQuantizerIndexConverter\n", + "\n", + "CODEC_MODEL_PATH = \"/home/vklimkov/workspace/emp/ckpt/easymagpietts_NEXT/25fps_spectral_codec_with_bandwidth_extension.nemo\"\n", + "EASYMAGPIE_NEMO = \"/home/vklimkov/workspace/emp/ckpt/easymagpietts_NEXT/2605_NemotronTTS_V0.2/v2/2605_EMTTS_SmallMamba_Step150k_posttrained_epoch12.nemo\"\n", + "\n", + "# Load the codec once (drop the discriminator to save memory).\n", + "_codec_cfg = AudioCodecModel.restore_from(CODEC_MODEL_PATH, return_config=True)\n", + "if \"use_scl_loss\" in _codec_cfg:\n", + " _codec_cfg.use_scl_loss = False\n", + "codec_model = AudioCodecModel.restore_from(CODEC_MODEL_PATH, strict=False, override_config_path=_codec_cfg)\n", + "if hasattr(codec_model, \"discriminator\"):\n", + " del codec_model.discriminator\n", + "codec_model = codec_model.eval().to(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", + "codec_device = next(codec_model.parameters()).device\n", + "\n", + "# The model may use a regrouped FSQ token space; map it back to the codec's native\n", + "# space when they differ (config read from the source EasyMagpie .nemo, no weights).\n", + "_em_cfg = EasyMagpieTTSInferenceModel.restore_from(EASYMAGPIE_NEMO, return_config=True)\n", + "_vq_cfg = _em_cfg.get(\"vector_quantizer\")\n", + "if _vq_cfg is not None and instantiate(_vq_cfg).num_codebooks != codec_model.vector_quantizer.num_codebooks:\n", + " codec_converter = VectorQuantizerIndexConverter(\n", + " vector_quantizer_original=codec_model.vector_quantizer,\n", + " vector_quantizer_new=instantiate(_vq_cfg),\n", + " ).to(codec_device)\n", + "else:\n", + " codec_converter = None\n", + "print(f\"codec native codebooks : {codec_model.vector_quantizer.num_codebooks}\")\n", + "print(f\"codec token converter : {'enabled' if codec_converter is not None else 'not needed'}\")\n", + "\n", + "S = arch.frame_stacking_factor\n", + "C = arch.num_stacked_codebooks // S\n", + "sample_rate = int(codec_model.output_sample_rate)\n", + "\n", + "\n", + "def decode_codes_to_waveform(audio_codes: torch.Tensor):\n", + " \"\"\"Decode one (T, C*S) stacked-code sequence to a mono float32 waveform.\"\"\"\n", + " stacked = audio_codes.to(codec_device, torch.long).T.unsqueeze(0) # (1, C*S, T)\n", + " T_out = stacked.size(-1)\n", + " codes = stacked.view(1, C, S, T_out).permute(0, 1, 3, 2).reshape(1, C, T_out * S) # (1, C, T*S)\n", + " codes_len = torch.tensor([codes.size(-1)], device=codec_device, dtype=torch.long)\n", + "\n", + " MIN_LEN = 4\n", + " if int(codes_len.min()) < MIN_LEN:\n", + " codes = torch.nn.functional.pad(codes, (0, MIN_LEN - int(codes_len.min())), value=0)\n", + " codes_len = codes_len.clamp(min=MIN_LEN)\n", + " codes = codes.clamp_(0, arch.codebook_size - 1)\n", + "\n", + " with torch.no_grad(), torch.autocast(device_type=codec_device.type, dtype=torch.float32):\n", + " if codec_converter is not None:\n", + " codes = codec_converter.convert_new_to_original(audio_tokens=codes, audio_lens=codes_len)\n", + " audio, audio_len = codec_model.decode(tokens=codes, tokens_len=codes_len)\n", + " return audio[0, : int(audio_len[0])].detach().cpu().float().numpy()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0e6fad73", + "metadata": {}, + "outputs": [], + "source": [ + "# Vocode and play each run.\n", + "for name, codes in [(\"whole-text (§4)\", audio_codes), (\"streamed text (§5)\", audio_codes_stream)]:\n", + " wav = decode_codes_to_waveform(codes)\n", + " print(f\"{name:<20}: {tuple(codes.shape)} codes -> {wav.shape[0] / sample_rate:.2f}s @ {sample_rate} Hz\")\n", + " display(Audio(wav, rate=sample_rate))" + ] + }, + { + "cell_type": "markdown", + "id": "d670e328", + "metadata": {}, + "source": [ + "### Streamed-text audio codes" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "77b12390", + "metadata": {}, + "outputs": [], + "source": [ + "import matplotlib.pylab as plt\n", + "\n", + "plt.figure(figsize=(10, 3))\n", + "plt.imshow(audio_codes_stream.T, aspect=\"auto\", interpolation=\"nearest\")\n", + "plt.title(f\"streamed-text audio codes ({audio_codes_stream.shape[0]} frames)\")\n", + "plt.xlabel(\"decode frame\")\n", + "plt.ylabel(\"stacked codebook\")\n", + "plt.colorbar()\n", + "plt.show()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "emp", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.13" + } }, - { - "cell_type": "code", - "execution_count": null, - "id": "c9a71b74", - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "\n", - "os.environ.setdefault(\"VLLM_WORKER_MULTIPROC_METHOD\", \"spawn\")\n", - "\n", - "import json\n", - "import tempfile\n", - "import uuid\n", - "from pathlib import Path\n", - "\n", - "import torch\n", - "import yaml\n", - "\n", - "from vllm import SamplingParams\n", - "from vllm.sampling_params import RequestOutputKind\n", - "from vllm_omni import AsyncOmni\n", - "\n", - "from easymagpie_vllm_omni.config import EasyMagpieOmniArch\n", - "\n", - "print(\"torch:\", torch.__version__, \"| cuda:\", torch.cuda.is_available())" - ] - }, - { - "cell_type": "markdown", - "id": "f7ff55fe", - "metadata": {}, - "source": [ - "## 1. Converted model directory\n", - "\n", - "`MODEL_DIR` is the directory written by the converter: `config.json`, weights, the\n", - "text tokenizer, and per-speaker embeddings. The engine loads everything from it;\n", - "here we only read a few config scalars used to build the prompt." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3e0df89e", - "metadata": {}, - "outputs": [], - "source": [ - "MODEL_DIR = Path(\"/home/vklimkov/workspace/emp/NeMo/examples/tts/easymagpie_vllm_omni/easymp_vllm_model\")\n", - "assert (MODEL_DIR / \"config.json\").exists(), f\"No config.json under {MODEL_DIR}; run the converter first.\"\n", - "\n", - "config = json.loads((MODEL_DIR / \"config.json\").read_text())\n", - "arch = EasyMagpieOmniArch.from_hf_config(type(\"Cfg\", (), config))\n", - "\n", - "TEXT_VOCAB = int(config[\"text_vocab_size\"])\n", - "TEXT_EOS_ID = TEXT_VOCAB - 2\n", - "AUDIO_STOP_TOKEN_ID = max(1, int(config.get(\"vocab_size\", 2)) - 1)\n", - "SPEECH_DELAY = int(getattr(arch, \"streaming_speech_delay\", 0) or 0)\n", - "\n", - "print(f\"Model dir : {MODEL_DIR}\")\n", - "print(f\"num_stacked_codebooks : {arch.num_stacked_codebooks} (C*S)\")\n", - "print(f\"audio_bos / audio_eos id : {arch.audio_bos_id} / {arch.audio_eos_id}\")\n", - "print(f\"text_vocab / text_eos : {TEXT_VOCAB} / {TEXT_EOS_ID}\")\n", - "print(f\"audio-EOS stop token id : {AUDIO_STOP_TOKEN_ID}\")\n", - "print(f\"streaming speech delay : {SPEECH_DELAY} frames\")" - ] - }, - { - "cell_type": "markdown", - "id": "012df58d", - "metadata": {}, - "source": [ - "## 2. Single-stage `AsyncOmni` engine\n", - "\n", - "One `llm` stage running the EasyMagpie talker (`worker_type=\"ar\"`).\n", - "`final_output_type=\"audio\"` makes the runner attach the per-step `audio_codes`\n", - "payload to each output. `async_chunk=True` lets the streaming-text path (§5) feed\n", - "one subword per step and is a no-op for the whole-text path (§4), so a single\n", - "engine serves both." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5085e9a4", - "metadata": {}, - "outputs": [], - "source": [ - "DECODE_STEPS = 256 # max audio frames to decode (trimmed at audio EOS)\n", - "MAX_MODEL_LEN = 1024\n", - "MAX_NUM_BATCHED_TOKENS = 1024\n", - "\n", - "stage_cfg = {\n", - " \"async_chunk\": True,\n", - " \"stage_args\": [\n", - " {\n", - " \"stage_id\": 0,\n", - " \"stage_type\": \"llm\",\n", - " \"is_comprehension\": True,\n", - " \"final_output\": True,\n", - " \"final_output_type\": \"audio\",\n", - " \"runtime\": {\"devices\": \"0\"},\n", - " \"engine_args\": {\n", - " \"model_stage\": \"easymagpie\",\n", - " \"max_num_seqs\": 1,\n", - " \"model_arch\": \"EasyMagpieTTSForConditionalGeneration\",\n", - " \"worker_type\": \"ar\",\n", - " \"scheduler_cls\": \"easymagpie_vllm_omni.scheduler.EasyMagpieARAsyncScheduler\",\n", - " \"trust_remote_code\": True,\n", - " \"async_scheduling\": True,\n", - " \"enable_prefix_caching\": False,\n", - " \"enforce_eager\": False,\n", - " \"engine_output_type\": \"audio\",\n", - " \"gpu_memory_utilization\": 0.8,\n", - " \"distributed_executor_backend\": \"uni\",\n", - " \"max_num_batched_tokens\": MAX_NUM_BATCHED_TOKENS,\n", - " \"max_model_len\": MAX_MODEL_LEN,\n", - " \"dtype\": \"float16\",\n", - " \"mamba_ssm_cache_dtype\": \"float32\",\n", - " \"attention_backend\": \"TRITON_ATTN\",\n", - " \"skip_tokenizer_init\": True,\n", - " },\n", - " \"default_sampling_params\": {\n", - " \"temperature\": 0.0,\n", - " \"max_tokens\": DECODE_STEPS,\n", - " \"detokenize\": False,\n", - " \"ignore_eos\": True,\n", - " \"stop_token_ids\": [AUDIO_STOP_TOKEN_ID],\n", - " },\n", - " }\n", - " ],\n", - "}\n", - "\n", - "_tmp = tempfile.NamedTemporaryFile(\n", - " mode=\"w\", suffix=\".yaml\", prefix=\"easymagpie_omni_demo_\", delete=False,\n", - ")\n", - "yaml.dump(stage_cfg, _tmp, sort_keys=False)\n", - "_tmp.close()\n", - "STAGE_CFG_PATH = _tmp.name\n", - "print(f\"Stage config: {STAGE_CFG_PATH}\")\n", - "\n", - "omni = AsyncOmni(\n", - " model=str(MODEL_DIR),\n", - " stage_configs_path=STAGE_CFG_PATH,\n", - " log_stats=False,\n", - " stage_init_timeout=300,\n", - ")\n", - "print(\"Engine ready (single stage: EasyMagpie talker)\")" - ] - }, - { - "cell_type": "markdown", - "id": "2736b86d", - "metadata": {}, - "source": [ - "## 3. Build the prompt\n", - "\n", - "Per-request inputs passed via `additional_information`: `speaker_embedding`\n", - "`(T_audio, embedding_dim)`, the `context_text` string, and the target `text` (all\n", - "tokenized in-engine). `prompt_token_ids` are placeholders sized by\n", - "`estimate_prompt_len(...)` to match the assembled prefill context. Audio sampling\n", - "`temperature` / `top_k` are forwarded to the local transformer." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "697c74b3", - "metadata": {}, - "outputs": [], - "source": [ - "torch.manual_seed(0)\n", - "\n", - "from transformers import AutoTokenizer\n", - "\n", - "from easymagpie_vllm_omni.easymagpie import EasyMagpieTTSForConditionalGeneration\n", - "\n", - "SPEAKER_NAME = \"eng\"\n", - "_loaded = torch.load(MODEL_DIR / \"speaker_embeddings\" / f\"{SPEAKER_NAME}.pt\", map_location=\"cpu\")\n", - "speaker_embedding = _loaded[\"speaker_encoding\"] if isinstance(_loaded, dict) else _loaded\n", - "speaker_embedding = speaker_embedding.to(torch.float32)\n", - "\n", - "CONTEXT_TEXT = \"[EN]\"\n", - "TEXT = \"Hello, this is a test of the EasyMagpie text to speech model.\"\n", - "\n", - "# Audio sampling (local transformer); temperature=0 == argmax (deterministic).\n", - "LT_TEMPERATURE = 0.0\n", - "LT_TOPK = 80\n", - "\n", - "# Same tokenizer the engine loads from MODEL_DIR; used only to size the placeholders.\n", - "tokenizer = AutoTokenizer.from_pretrained(MODEL_DIR, trust_remote_code=True)\n", - "prompt_len = EasyMagpieTTSForConditionalGeneration.estimate_prompt_len(\n", - " speaker_embedding,\n", - " tokenize=lambda t: tokenizer.encode(t),\n", - " context_text=CONTEXT_TEXT,\n", - " has_task_embedding=arch.num_task_embeddings > 0,\n", - ")\n", - "\n", - "prompt = {\n", - " \"prompt_token_ids\": [0] * prompt_len,\n", - " \"additional_information\": {\n", - " \"speaker_embedding\": speaker_embedding,\n", - " \"context_text\": CONTEXT_TEXT,\n", - " \"text\": TEXT,\n", - " \"temperature\": LT_TEMPERATURE,\n", - " \"top_k\": LT_TOPK,\n", - " },\n", - "}\n", - "assert prompt_len + DECODE_STEPS <= MAX_MODEL_LEN\n", - "\n", - "# output_kind=DELTA: audio_codes arrives as a growing list during decode\n", - "# ([prefill_prefix, frame_0, frame_1, ...]); prefill/final steps yield a tensor.\n", - "sampling_params = SamplingParams(\n", - " temperature=0.0,\n", - " max_tokens=DECODE_STEPS,\n", - " detokenize=False,\n", - " ignore_eos=True,\n", - " stop_token_ids=[AUDIO_STOP_TOKEN_ID],\n", - " output_kind=RequestOutputKind.DELTA,\n", - ")\n", - "\n", - "print(f\"speaker_embedding : {tuple(speaker_embedding.shape)}\")\n", - "print(f\"context_text / text : {CONTEXT_TEXT!r} / {TEXT!r}\")\n", - "print(f\"prompt_len (placeholders) : {prompt_len}\")" - ] - }, - { - "cell_type": "markdown", - "id": "3ef8934d", - "metadata": {}, - "source": [ - "## 4. Whole-text inference -> audio codes\n", - "\n", - "`omni.generate(...)` yields one `RequestOutput` per engine step, each carrying the\n", - "**cumulative** `multimodal_output[\"audio_codes\"]` tensor: the first `prompt_len`\n", - "rows are the prefill prefix, followed by one `(1, C*S)` decoded frame per step\n", - "(occasionally surfaced as a list of per-step tensors, which we just concatenate).\n", - "We keep the largest tensor seen, drop the prefill prefix, then drop the\n", - "`speech_delay` warm-up frames and the trailing audio-EOS frame." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6d0ccbd4", - "metadata": {}, - "outputs": [], - "source": [ - "async def run_request(prompt, sampling_params):\n", - " \"\"\"Keep the cumulative audio-code tensor [prefill_prefix | decoded frames].\n", - "\n", - " Each step exposes audio_codes as a growing tensor (sometimes a list of per-step\n", - " tensors, which we concatenate); we keep the largest one seen.\n", - " \"\"\"\n", - " request_id = f\"easymagpie-{uuid.uuid4().hex[:8]}\"\n", - " codes = None\n", - " async for out in omni.generate(prompt, sampling_params_list=[sampling_params], request_id=request_id):\n", - " payload = (out.multimodal_output or {}).get(\"audio_codes\")\n", - " if isinstance(payload, list):\n", - " payload = torch.cat([t for t in payload if isinstance(t, torch.Tensor)], dim=0) if payload else None\n", - " if isinstance(payload, torch.Tensor) and (codes is None or payload.shape[0] >= codes.shape[0]):\n", - " codes = payload.detach().cpu().to(torch.long)\n", - " return codes\n", - "\n", - "\n", - "codes = await run_request(prompt, sampling_params)\n", - "# Rows 0:prompt_len are the prefill prefix; decoded frames follow. Drop the prefix,\n", - "# the speech-delay warm-up frames, and the trailing audio-EOS frame.\n", - "audio_codes = codes[prompt_len + SPEECH_DELAY : -1]\n", - "print(f\"cumulative codes : {tuple(codes.shape)} (prompt_len={prompt_len})\")\n", - "print(f\"audio_codes : {tuple(audio_codes.shape)} (dropped prefix + {SPEECH_DELAY} warm-up + 1 EOS)\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "04196662", - "metadata": {}, - "outputs": [], - "source": [ - "import matplotlib.pylab as plt\n", - "\n", - "plt.figure(figsize=(10, 3))\n", - "plt.imshow(audio_codes.T, aspect=\"auto\", interpolation=\"nearest\")\n", - "plt.title(f\"whole-text audio codes ({audio_codes.shape[0]} frames)\")\n", - "plt.xlabel(\"decode frame\")\n", - "plt.ylabel(\"stacked codebook\")\n", - "plt.colorbar()\n", - "plt.show()" - ] - }, - { - "cell_type": "markdown", - "id": "a32b07d5", - "metadata": {}, - "source": [ - "## 5. Streaming text input (one subword per decode step)\n", - "\n", - "Same model, but push subword ids **one at a time** as it decodes (the live-client\n", - "path). `omni.generate(...)` is given an async generator of `StreamingInput` chunks:\n", - "chunk 0 is the prefill (speaker + context, no text), each later chunk carries one\n", - "`text_token` with `max_tokens=1` (one chunk -> one decoded frame). After the text\n", - "is exhausted we feed the `-1` mask sentinel until the model emits the audio-EOS\n", - "token. Input is paced by output via `go_queue`. Codes are collected exactly as in\n", - "§4 (keep the cumulative tensor, slice off prefix + warm-up + EOS)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "aa57a573", - "metadata": {}, - "outputs": [], - "source": [ - "import asyncio\n", - "from collections.abc import AsyncGenerator\n", - "\n", - "try:\n", - " from vllm.engine.protocol import StreamingInput\n", - "except ImportError:\n", - " from vllm.v1.engine.async_llm import StreamingInput\n", - "\n", - "# Tokenize the target text the same way the model does (specials off + text-EOS).\n", - "text_ids = list(tokenizer.encode(TEXT, add_special_tokens=False)) + [TEXT_EOS_ID]\n", - "print(f\"streamed text ids ({len(text_ids)}): {text_ids}\")\n", - "\n", - "stream_params = SamplingParams(\n", - " temperature=0.0,\n", - " max_tokens=1,\n", - " detokenize=False,\n", - " ignore_eos=True,\n", - " stop_token_ids=[AUDIO_STOP_TOKEN_ID],\n", - " output_kind=RequestOutputKind.DELTA,\n", - ")\n", - "\n", - "go_queue: asyncio.Queue[bool] = asyncio.Queue()\n", - "\n", - "\n", - "async def stream_text_inputs() -> AsyncGenerator[StreamingInput, None]:\n", - " # Prefill: speaker + context, NO text (its absence selects streaming-text mode).\n", - " # The first subword rides on the first decode chunk, not the prefill.\n", - " prefill_info = {\n", - " \"speaker_embedding\": speaker_embedding,\n", - " \"context_text\": CONTEXT_TEXT,\n", - " \"temperature\": LT_TEMPERATURE,\n", - " \"top_k\": LT_TOPK,\n", - " }\n", - " yield StreamingInput(\n", - " prompt={\"prompt_token_ids\": [0] * prompt_len, \"additional_information\": prefill_info},\n", - " sampling_params=stream_params,\n", - " )\n", - " # One chunk per decode frame, paced by go_queue: text_ids[k] on decode step k,\n", - " # then the -1 mask once the text is exhausted.\n", - " step = 0\n", - " while True:\n", - " await go_queue.get()\n", - " tok = int(text_ids[step]) if step < len(text_ids) else -1\n", - " yield StreamingInput(\n", - " prompt={\"prompt_token_ids\": [0], \"additional_information\": {\"text_token\": tok}},\n", - " sampling_params=stream_params,\n", - " )\n", - " step += 1\n", - "\n", - "\n", - "async def run_streaming_request():\n", - " \"\"\"Collect codes like §4, pacing one chunk per decoded frame.\n", - "\n", - " Every streaming segment reports finish_reason \"length\"; the request truly ends\n", - " only at the audio-EOS stop token, where we stop pacing and break.\n", - " \"\"\"\n", - " request_id = f\"easymagpie-stream-{uuid.uuid4().hex[:8]}\"\n", - " codes = None\n", - " MAX_STEPS = 4 * DECODE_STEPS + 16\n", - " steps = 0\n", - " async for out in omni.generate(\n", - " stream_text_inputs(), sampling_params_list=[stream_params], request_id=request_id\n", - " ):\n", - " steps += 1\n", - " co = out.outputs[0] if out.outputs else None\n", - " payload = (out.multimodal_output or {}).get(\"audio_codes\")\n", - " if isinstance(payload, list):\n", - " payload = torch.cat([t for t in payload if isinstance(t, torch.Tensor)], dim=0) if payload else None\n", - " if isinstance(payload, torch.Tensor) and (codes is None or payload.shape[0] >= codes.shape[0]):\n", - " codes = payload.detach().cpu().to(torch.long)\n", - " if getattr(co, \"stop_reason\", None) == AUDIO_STOP_TOKEN_ID or steps >= MAX_STEPS:\n", - " break\n", - " await go_queue.put(True)\n", - " return codes\n", - "\n", - "\n", - "codes_stream = await run_streaming_request()\n", - "audio_codes_stream = codes_stream[prompt_len + SPEECH_DELAY : -1]\n", - "print(f\"cumulative codes : {tuple(codes_stream.shape)}\")\n", - "print(f\"audio_codes : {tuple(audio_codes_stream.shape)}\")" - ] - }, - { - "cell_type": "markdown", - "id": "4d86ebf5", - "metadata": {}, - "source": [ - "## 6. Decode codes to waveforms\n", - "\n", - "The engine emits **stacked** codes `(T, C*S)` with `C*S = 16`. To vocode we mirror\n", - "`EasyMagpieTTSInferenceModel`: load the `.nemo` codec, unstack `(T, C*S)` ->\n", - "`(1, C, T*S)`, optionally remap the regrouped FSQ token space back to the codec's\n", - "native space, then `codec_model.decode(...)`.\n", - "\n", - "Set `CODEC_MODEL_PATH` / `EASYMAGPIE_NEMO` to the same `.nemo` files passed to the\n", - "converter. Needs NeMo importable in this environment." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3ab9fedc", - "metadata": {}, - "outputs": [], - "source": [ - "from hydra.utils import instantiate\n", - "from IPython.display import Audio, display\n", - "\n", - "from nemo.collections.tts.models import AudioCodecModel\n", - "from nemo.collections.tts.models.easy_magpietts_inference import EasyMagpieTTSInferenceModel\n", - "from nemo.collections.tts.modules.audio_codec_modules import VectorQuantizerIndexConverter\n", - "\n", - "CODEC_MODEL_PATH = \"/home/vklimkov/workspace/emp/ckpt/easymagpietts_NEXT/25fps_spectral_codec_with_bandwidth_extension.nemo\"\n", - "EASYMAGPIE_NEMO = \"/home/vklimkov/workspace/emp/ckpt/easymagpietts_NEXT/2605_NemotronTTS_V0.2/v2/2605_EMTTS_SmallMamba_Step150k_posttrained_epoch12.nemo\"\n", - "\n", - "# Load the codec once (drop the discriminator to save memory).\n", - "_codec_cfg = AudioCodecModel.restore_from(CODEC_MODEL_PATH, return_config=True)\n", - "if \"use_scl_loss\" in _codec_cfg:\n", - " _codec_cfg.use_scl_loss = False\n", - "codec_model = AudioCodecModel.restore_from(CODEC_MODEL_PATH, strict=False, override_config_path=_codec_cfg)\n", - "if hasattr(codec_model, \"discriminator\"):\n", - " del codec_model.discriminator\n", - "codec_model = codec_model.eval().to(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", - "codec_device = next(codec_model.parameters()).device\n", - "\n", - "# The model may use a regrouped FSQ token space; map it back to the codec's native\n", - "# space when they differ (config read from the source EasyMagpie .nemo, no weights).\n", - "_em_cfg = EasyMagpieTTSInferenceModel.restore_from(EASYMAGPIE_NEMO, return_config=True)\n", - "_vq_cfg = _em_cfg.get(\"vector_quantizer\")\n", - "if _vq_cfg is not None and instantiate(_vq_cfg).num_codebooks != codec_model.vector_quantizer.num_codebooks:\n", - " codec_converter = VectorQuantizerIndexConverter(\n", - " vector_quantizer_original=codec_model.vector_quantizer,\n", - " vector_quantizer_new=instantiate(_vq_cfg),\n", - " ).to(codec_device)\n", - "else:\n", - " codec_converter = None\n", - "print(f\"codec native codebooks : {codec_model.vector_quantizer.num_codebooks}\")\n", - "print(f\"codec token converter : {'enabled' if codec_converter is not None else 'not needed'}\")\n", - "\n", - "S = arch.frame_stacking_factor\n", - "C = arch.num_stacked_codebooks // S\n", - "sample_rate = int(codec_model.output_sample_rate)\n", - "\n", - "\n", - "def decode_codes_to_waveform(audio_codes: torch.Tensor):\n", - " \"\"\"Decode one (T, C*S) stacked-code sequence to a mono float32 waveform.\"\"\"\n", - " stacked = audio_codes.to(codec_device, torch.long).T.unsqueeze(0) # (1, C*S, T)\n", - " T_out = stacked.size(-1)\n", - " codes = stacked.view(1, C, S, T_out).permute(0, 1, 3, 2).reshape(1, C, T_out * S) # (1, C, T*S)\n", - " codes_len = torch.tensor([codes.size(-1)], device=codec_device, dtype=torch.long)\n", - "\n", - " MIN_LEN = 4\n", - " if int(codes_len.min()) < MIN_LEN:\n", - " codes = torch.nn.functional.pad(codes, (0, MIN_LEN - int(codes_len.min())), value=0)\n", - " codes_len = codes_len.clamp(min=MIN_LEN)\n", - " codes = codes.clamp_(0, arch.codebook_size - 1)\n", - "\n", - " with torch.no_grad(), torch.autocast(device_type=codec_device.type, dtype=torch.float32):\n", - " if codec_converter is not None:\n", - " codes = codec_converter.convert_new_to_original(audio_tokens=codes, audio_lens=codes_len)\n", - " audio, audio_len = codec_model.decode(tokens=codes, tokens_len=codes_len)\n", - " return audio[0, : int(audio_len[0])].detach().cpu().float().numpy()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "0e6fad73", - "metadata": {}, - "outputs": [], - "source": [ - "# Vocode and play each run.\n", - "for name, codes in [(\"whole-text (§4)\", audio_codes), (\"streamed text (§5)\", audio_codes_stream)]:\n", - " wav = decode_codes_to_waveform(codes)\n", - " print(f\"{name:<20}: {tuple(codes.shape)} codes -> {wav.shape[0] / sample_rate:.2f}s @ {sample_rate} Hz\")\n", - " display(Audio(wav, rate=sample_rate))" - ] - }, - { - "cell_type": "markdown", - "id": "d670e328", - "metadata": {}, - "source": [ - "### Streamed-text audio codes" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "77b12390", - "metadata": {}, - "outputs": [], - "source": [ - "import matplotlib.pylab as plt\n", - "\n", - "plt.figure(figsize=(10, 3))\n", - "plt.imshow(audio_codes_stream.T, aspect=\"auto\", interpolation=\"nearest\")\n", - "plt.title(f\"streamed-text audio codes ({audio_codes_stream.shape[0]} frames)\")\n", - "plt.xlabel(\"decode frame\")\n", - "plt.ylabel(\"stacked codebook\")\n", - "plt.colorbar()\n", - "plt.show()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "emp", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.13" - } - }, - "nbformat": 4, - "nbformat_minor": 5 + "nbformat": 4, + "nbformat_minor": 5 } diff --git a/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/scheduler.py b/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/scheduler.py new file mode 100644 index 000000000000..6f82bb23332e --- /dev/null +++ b/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/scheduler.py @@ -0,0 +1,75 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Streaming-aware scheduler for the single-stage EasyMagpieTTS engine. + +vLLM-Omni's stage-0 streaming session update +(:meth:`OmniARScheduler._update_request_as_session`) extends the prompt token +ids from each ``StreamingInput`` chunk but **never updates** +``session.additional_information``. For EasyMagpie's streaming-text path that +silently drops the per-chunk ``text_token`` payload on the scheduler side: the +runner only ever sees the initial request's ``additional_information``, so every +decode step reads ``text_token=None``, the text channel is masked off, and the +model emits audio-EOS almost immediately (a handful of frames instead of the +full utterance). + +:class:`EasyMagpieARAsyncScheduler` restores the missing propagation. It is a +drop-in replacement for ``OmniARAsyncScheduler``; wire it in via the stage's +``scheduler_cls``:: + + "scheduler_cls": "easymagpie_vllm_omni.scheduler.EasyMagpieARAsyncScheduler" +""" +from __future__ import annotations + +from vllm.v1.request import Request, StreamingUpdate + +from vllm_omni.core.sched.omni_ar_scheduler import OmniARAsyncScheduler + + +class EasyMagpieARAsyncScheduler(OmniARAsyncScheduler): + """``OmniARAsyncScheduler`` that forwards per-chunk ``additional_information``. + + Replace (not merge) is the correct session-level semantics: the session field + is just a courier for the latest chunk's payload to ``OmniNewRequestData``. + Per-key accumulation, where a model needs it, is handled by the runner's + ``_update_streaming_input_additional_info`` against the model's + ``streaming_accumulated_keys`` set, so the merge policy stays a per-model + concern. ``None`` is treated as "this chunk omitted the field" (keep the prior + value) rather than "clear the session", so a client may keep pumping + placeholder chunks (e.g. the masking ``text_token=-1`` sentinel still sets a + value; a truly empty chunk leaves the previous payload intact). + """ + + def _update_request_as_session(self, session: Request, update: StreamingUpdate) -> None: + super()._update_request_as_session(session, update) + + # ``check_stop`` decides segment termination on ``session.max_tokens``, + # a value cached once at request creation. The base session update swaps + # ``session.sampling_params`` for each chunk but never refreshes the + # cached ``session.max_tokens`` (even though ``StreamingUpdate`` carries + # one). Without this, a chunk that raises ``max_tokens`` — e.g. handing + # the request off to a free-running acoustic tail once the text stream is + # exhausted — is silently capped at the request's *initial* ``max_tokens`` + # (1 in the one-frame-per-chunk streaming-text path), so every segment, + # the tail included, stops after a single decoded frame. + new_max_tokens = getattr(update, "max_tokens", None) + if new_max_tokens is not None: + session.max_tokens = new_max_tokens + + # At stage_id != 0 the base class already routed through + # ``_replace_session_with_streaming_update`` (which sets + # ``additional_information``); only stage 0 drops it. + if self.vllm_config.model_config.stage_id == 0: + new_info = getattr(update, "additional_information", None) + if new_info is not None: + session.additional_information = new_info From bafae309c88a14f44487a814f8d88e39e0e502db Mon Sep 17 00:00:00 2001 From: Viacheslav Klimkov Date: Wed, 10 Jun 2026 01:21:00 +0200 Subject: [PATCH 31/45] examples/tts/easymagpie_vllm_omni/benchmark_model.py: add benchmarking of streaming mode, simplify Signed-off-by: Viacheslav Klimkov --- .../easymagpie_vllm_omni/benchmark_model.py | 1156 +++++------------ 1 file changed, 337 insertions(+), 819 deletions(-) diff --git a/examples/tts/easymagpie_vllm_omni/benchmark_model.py b/examples/tts/easymagpie_vllm_omni/benchmark_model.py index da40381776ba..770c034bcee6 100644 --- a/examples/tts/easymagpie_vllm_omni/benchmark_model.py +++ b/examples/tts/easymagpie_vllm_omni/benchmark_model.py @@ -13,59 +13,24 @@ # limitations under the License. """Benchmark the EasyMagpieTTS talker via a single-stage AsyncOmni engine. -Runs the EasyMagpie talker (``EasyMagpieTTSForConditionalGeneration``) only — -no codec / code2wav — producing stacked audio codes as output. It mirrors the -reference ``qwen3-tts`` talker benchmark and the -``easymagpie_inference_demo.ipynb`` engine setup. +Two input modes, selectable with ``--streaming``: -Metrics measured under configurable concurrency: +* whole-text (default) — the full target text is handed to the engine up front. +* streaming-text — subword ids are pushed one at a time as the model decodes + (prefill chunk, then one ``StreamingInput`` chunk per subword with + ``max_tokens=1``, then a free-running acoustic tail). -* **TTFT** — time to first decoded frame (first engine token). -* **ITL** — per-token inter-token latency (excluding the first token). -* **E2E** — end-to-end latency per request (up to the audio-EOS frame). -* **RTX** — real-time factor (generated audio seconds / wall time). Both the - per-request RTX and an overall (concurrency-aware) RTX are reported. -* **Throughput** — frames/s and requests/s. - -The decode loop stops at the audio-EOS frame (the EasyMagpie model signals -end-of-speech inside codebook 0 of the codes, not via the vLLM token stream), -so E2E / RTX reflect the real synthesized length rather than the full token -budget. Audio duration is derived from the number of decoded frames: -``audio_seconds = (frames - speech_delay) * frame_stacking_factor / codec_fps``. - -Reads texts from a file (one utterance per line, optionally tab-separated with -the text in the second column) or uses a small built-in default set. +Both run on the same engine config. Reports throughput, TTFT, ITL (mean + p95), +EOS hit rate and overall RTF. Usage: - # Basic benchmark with default prompts - python benchmark_easymagpie_tts.py \\ - --model ./easymp_vllm_model \\ - --num-requests 50 - - # From a text file with a concurrency sweep - python benchmark_easymagpie_tts.py \\ - --model ./easymp_vllm_model \\ - --text-file texts.txt \\ - --num-requests 100 \\ - --concurrency 1 4 8 - - # With torch profiler on the run - python benchmark_easymagpie_tts.py \\ - --model ./easymp_vllm_model \\ - --num-requests 20 --concurrency 1 --profile - - # Save JSON results - python benchmark_easymagpie_tts.py \\ - --model ./easymp_vllm_model \\ - --text-file texts.txt \\ - --num-requests 100 --concurrency 1 4 \\ - --result-dir results/ + python benchmark_model.py --model ./easymp_vllm_model --num-requests 50 + python benchmark_model.py --model ./easymp_vllm_model -n 50 --streaming + python benchmark_model.py --model ./easymp_vllm_model -n 50 -c 1 4 8 """ import os -# Keep spawn semantics consistent with the qwen3-tts / eartts demos in case the -# executor backend is switched to a multiproc one. os.environ.setdefault("VLLM_WORKER_MULTIPROC_METHOD", "spawn") import argparse @@ -75,20 +40,30 @@ import tempfile import time import uuid -from dataclasses import asdict, dataclass, field -from datetime import datetime +from dataclasses import dataclass, field from pathlib import Path from typing import Any, Optional import numpy as np import yaml -logging.basicConfig( - level=logging.INFO, - format="%(asctime)s %(levelname)s %(name)s: %(message)s", -) +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s") logger = logging.getLogger(__name__) +# ── Hardcoded run settings ───────────────────────────────────────────────── +SPEAKER = "eng" +CONTEXT_TEXT = "[EN]" +LT_TEMPERATURE = 0.7 # audio (local-transformer) sampling temperature +LT_TOPK = 80 # audio sampling top-k +CODEC_FRAME_RATE = 25.0 # Hz, used to convert decoded frames -> audio seconds (RTF) +GPU_MEMORY_UTILIZATION = 0.8 +DISTRIBUTED_EXECUTOR_BACKEND = "uni" +ENFORCE_EAGER = False +DTYPE = "float16" +STAGE_INIT_TIMEOUT = 300 +# vLLM CUDA-graph capture strategy; None == vLLM default (FULL_AND_PIECEWISE). +CUDAGRAPH_MODE: Optional[str] = None + DEFAULT_PROMPTS = [ "Hello, welcome to the voice synthesis benchmark test.", "She said she would be here by noon, but nobody showed up.", @@ -100,175 +75,107 @@ "After the meeting, we should discuss the quarterly results and plan for the next phase.", "Learning a new language takes patience, practice, and a genuine curiosity about other cultures.", "The train leaves at half past seven, so we need to arrive at the station before then.", - "Could you please turn down the music a little bit, I'm trying to concentrate on my work.", - "It was a dark and stormy night when the old lighthouse keeper heard a knock at the door.", ] # --------------------------------------------------------------------------- -# Stage config generation +# Stage config # --------------------------------------------------------------------------- -def _build_easymagpie_stage_config( - max_num_seqs: int = 1, - profile: bool = False, - torch_profiler_dir: str = "./profiler_traces", - with_stack: bool = False, - record_shapes: bool = False, - gpu_memory_utilization: float = 0.5, - max_model_len: int = 1024, - max_num_batched_tokens: int = 1024, - enforce_eager: bool = False, - max_new_tokens: int = 256, - dtype: str = "float16", - distributed_executor_backend: str = "uni", - cudagraph_mode: Optional[str] = None, - load_format: Optional[str] = None, +def _build_stage_config( + max_num_seqs: int, + max_model_len: int, + max_num_batched_tokens: int, + max_new_tokens: int, + profile: bool, + torch_profiler_dir: str, + load_format: Optional[str], ) -> dict: - """Build a single-stage YAML dict containing only the EasyMagpie talker. - - Mirrors the engine_args used in ``easymagpie_inference_demo.ipynb``. - - ``cudagraph_mode`` (when set and ``enforce_eager`` is False) selects the - vLLM CUDA-graph capture strategy via ``compilation_config.cudagraph_mode``: - - * ``FULL_AND_PIECEWISE`` (vLLM default) — a single full graph over the whole - forward for uniform/decode-only batches, piecewise (per compile group: - backbone vs local transformer) for mixed/prefill batches. - * ``PIECEWISE`` — always piecewise, so the backbone and local transformer are - captured as *separate* graphs even during decode. This re-introduces a - launch boundary between them (so decode is a touch slower than FULL), but - makes the backbone-vs-LT split visible as two distinct ``cudaGraphLaunch`` - events in a profiler. - * ``FULL`` / ``FULL_DECODE_ONLY`` — full graph (decode only) capture. - * ``NONE`` — no CUDA graphs (equivalent to ``--enforce-eager``). - """ + """Single-stage YAML dict for the EasyMagpie talker (see the demo notebook).""" engine_args: dict[str, Any] = { "model_stage": "easymagpie", "max_num_seqs": max_num_seqs, "model_arch": "EasyMagpieTTSForConditionalGeneration", "worker_type": "ar", - "scheduler_cls": "vllm_omni.core.sched.omni_ar_scheduler.OmniARAsyncScheduler", - "enforce_eager": enforce_eager, + # EasyMagpie-aware scheduler serves both paths: it forwards per-chunk + # text_token and the raised acoustic-tail max_tokens for streaming, and is + # a drop-in equivalent of the stock scheduler for whole-text. + "scheduler_cls": "easymagpie_vllm_omni.scheduler.EasyMagpieARAsyncScheduler", + "enforce_eager": ENFORCE_EAGER, "trust_remote_code": True, "async_scheduling": True, "enable_prefix_caching": False, "engine_output_type": "audio", - "gpu_memory_utilization": gpu_memory_utilization, - # "uni" runs the worker in-process (no shm_broadcast IPC); use "mp" - # only when TP/PP > 1 or you actually need a separate worker process. - "distributed_executor_backend": distributed_executor_backend, + "gpu_memory_utilization": GPU_MEMORY_UTILIZATION, + "distributed_executor_backend": DISTRIBUTED_EXECUTOR_BACKEND, "max_num_batched_tokens": max_num_batched_tokens, "max_model_len": max_model_len, - # bf16/fp16 (not fp32): the Nemotron-H fused-MoE Triton kernel's block - # sizes are tuned for 16-bit and overflow shared memory in fp32. - "dtype": dtype, + "dtype": DTYPE, "mamba_ssm_cache_dtype": "float32", "attention_backend": "TRITON_ATTN", - # We feed prompt_token_ids directly; the model loads the bundled - # AutoTokenizer from the model dir to tokenize context_text + text. "skip_tokenizer_init": True, } - - # Weight loading strategy. ``dummy`` initializes random weights and skips the - # checkpoint entirely — pair it with a dummy config dir (e.g. a Qwen3 backbone - # profile) to benchmark a backbone architecture without a trained checkpoint. if load_format is not None: engine_args["load_format"] = load_format - - # CUDA-graph capture strategy. ``enforce_eager`` already disables graphs, so - # only set compilation_config when graphs are enabled (mirrors the sidecar - # server). Passed as a plain dict so it survives YAML serialization; vLLM - # parses it into a CompilationConfig. - if cudagraph_mode is not None and not enforce_eager: - engine_args["compilation_config"] = {"cudagraph_mode": cudagraph_mode} - + if CUDAGRAPH_MODE is not None and not ENFORCE_EAGER: + engine_args["compilation_config"] = {"cudagraph_mode": CUDAGRAPH_MODE} if profile: engine_args["profiler_config"] = { "profiler": "torch", "torch_profiler_dir": os.path.abspath(torch_profiler_dir), - "torch_profiler_with_stack": with_stack, - "torch_profiler_record_shapes": record_shapes, + "torch_profiler_with_stack": True, + "torch_profiler_record_shapes": True, } - cfg = { + return { + # async_chunk enables the streaming-text feed; no-op for whole-text. + "async_chunk": True, "stage_args": [ { "stage_id": 0, "stage_type": "llm", "is_comprehension": True, "final_output": True, - # "audio" (not "latent") is required for a single-stage AR TTS - # model: it makes the AR model runner attach the per-step - # multimodal payload ("audio_codes") to the output so the codes - # reach the client. "final_output_type": "audio", "runtime": {"devices": "0"}, "engine_args": engine_args, "default_sampling_params": { - # The backbone token sampler is a no-op (audio is sampled in - # the local transformer); the audio temperature/top-k are - # forwarded per-request via additional_information. "temperature": 0.0, "max_tokens": max_new_tokens, "detokenize": False, - # Audio EOS lives in the codes, not the vLLM token stream, so - # let the budget run and stop client-side at the EOS frame. "ignore_eos": True, }, } ], } - return cfg def _write_temp_stage_config(cfg: dict) -> str: - """Write stage config dict to a temp YAML file, return its path.""" - tmp = tempfile.NamedTemporaryFile( - mode="w", - suffix=".yaml", - prefix="easymagpie_bench_", - delete=False, - ) + tmp = tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", prefix="easymagpie_bench_", delete=False) yaml.dump(cfg, tmp, default_flow_style=False, sort_keys=False) tmp.close() - logger.info("Wrote single-stage config to %s", tmp.name) return tmp.name # --------------------------------------------------------------------------- -# Model metadata (arch scalars + tokenizer + speaker embedding) +# Model metadata # --------------------------------------------------------------------------- @dataclass class ModelMeta: - """Scalars + assets needed to build prompts and interpret outputs.""" - - arch: Any tokenizer: Any speaker_embedding: Any # torch.Tensor (T_audio, embedding_dim) prompt_len: int audio_eos_id: int speech_delay: int frame_stacking_factor: int - stop_token_id: int # backbone stop token the model emits at the audio-EOS frame - + stop_token_id: int # backbone token emitted at the audio-EOS frame + text_eos_id: int # appended to streamed subword ids -def _load_model_meta( - model_dir: str, - speaker: str, - speaker_embedding_path: Optional[str], - context_text: str, -) -> ModelMeta: - """Read config.json, tokenizer, and the speaker embedding from the model dir. - Mirrors the prompt-prep cells of ``easymagpie_inference_demo.ipynb``: the - arch scalars come from ``config.json``, the speaker embedding from - ``speaker_embeddings/.pt``, and the prefill placeholder length from - ``EasyMagpieTTSForConditionalGeneration.estimate_prompt_len(...)``. - """ +def _load_model_meta(model_dir: str) -> ModelMeta: import torch from transformers import AutoTokenizer @@ -279,31 +186,21 @@ def _load_model_meta( config = json.loads((model_path / "config.json").read_text()) arch = EasyMagpieOmniArch.from_hf_config(type("Cfg", (), config)) - # Speaker-encoded context audio (audio branch of prepare_context_tensors). - if speaker_embedding_path is not None: - emb_path = Path(speaker_embedding_path) - else: - emb_path = model_path / "speaker_embeddings" / f"{speaker}.pt" + emb_path = model_path / "speaker_embeddings" / f"{SPEAKER}.pt" if not emb_path.exists(): raise FileNotFoundError(f"Speaker embedding not found: {emb_path}") loaded = torch.load(emb_path, map_location="cpu") - speaker_embedding = loaded["speaker_encoding"] if isinstance(loaded, dict) else loaded - speaker_embedding = speaker_embedding.to(torch.float32) + speaker_embedding = (loaded["speaker_encoding"] if isinstance(loaded, dict) else loaded).to(torch.float32) tokenizer = AutoTokenizer.from_pretrained(model_dir, trust_remote_code=True) - - # prompt_len depends only on the speaker embedding + context_text (+ task - # embedding) — NOT on the target text (which is streamed in-engine), so we - # size it once. prompt_len = EasyMagpieTTSForConditionalGeneration.estimate_prompt_len( speaker_embedding, tokenize=lambda t: tokenizer.encode(t), - context_text=context_text, + context_text=CONTEXT_TEXT, has_task_embedding=arch.num_task_embeddings > 0, ) return ModelMeta( - arch=arch, tokenizer=tokenizer, speaker_embedding=speaker_embedding, prompt_len=int(prompt_len), @@ -311,27 +208,20 @@ def _load_model_meta( speech_delay=int(getattr(arch, "streaming_speech_delay", 0) or 0), frame_stacking_factor=int(arch.frame_stacking_factor), stop_token_id=EasyMagpieTTSForConditionalGeneration.audio_eos_stop_token_id(type("Cfg", (), config)), + text_eos_id=int(config.get("text_vocab_size", config.get("vocab_size", 0))) - 2, ) -def build_prompt( - text: str, - meta: ModelMeta, - context_text: str, - lt_temperature: float, - lt_topk: int, -) -> dict: - """Build an engine input dict from a target sentence + the shared assets.""" - additional_information = { - "speaker_embedding": meta.speaker_embedding, # (T_audio, embedding_dim) - "context_text": context_text, # plain string, tokenized in-model - "text": text, # plain target sentence, tokenized in-model - "temperature": lt_temperature, # audio sampling temperature (local transformer) - "top_k": lt_topk, # audio sampling top-k (local transformer) - } +def build_prompt(text: str, meta: ModelMeta) -> dict: return { "prompt_token_ids": [0] * meta.prompt_len, - "additional_information": additional_information, + "additional_information": { + "speaker_embedding": meta.speaker_embedding, + "context_text": CONTEXT_TEXT, + "text": text, + "temperature": LT_TEMPERATURE, + "top_k": LT_TOPK, + }, } @@ -343,211 +233,194 @@ def build_prompt( @dataclass class RequestResult: success: bool = False - text: str = "" - prompt_len: int = 0 - num_generated: int = 0 # decoded frames (engine tokens) up to EOS - delta_frames: int = 0 # decode frames consumed as per-step deltas (should match num_generated) - audio_frames: int = 0 # codec frames of real audio (post speech-delay, pre-EOS) - audio_s: float = 0.0 # synthesized audio duration in seconds - steps: int = 0 + audio_s: float = 0.0 eos_reached: bool = False ttft_s: float = 0.0 - e2e_s: float = 0.0 - rtx: float = 0.0 # audio_s / e2e_s inter_token_latencies: list = field(default_factory=list) error: str = "" -@dataclass -class BenchmarkResult: - config_name: str = "" - concurrency: int = 0 - num_requests: int = 0 - completed: int = 0 - failed: int = 0 - duration_s: float = 0.0 - # TTFT - mean_ttft_ms: float = 0.0 - median_ttft_ms: float = 0.0 - p95_ttft_ms: float = 0.0 - p99_ttft_ms: float = 0.0 - # E2E - mean_e2e_ms: float = 0.0 - median_e2e_ms: float = 0.0 - p95_e2e_ms: float = 0.0 - p99_e2e_ms: float = 0.0 - # ITL (inter-token latency, excluding first token) - mean_itl_ms: float = 0.0 - median_itl_ms: float = 0.0 - p95_itl_ms: float = 0.0 - p99_itl_ms: float = 0.0 - # RTX (real-time factor: synthesized audio seconds / generation seconds) - mean_rtx: float = 0.0 - median_rtx: float = 0.0 - overall_rtx: float = 0.0 # total_audio_s / wall_clock_duration (concurrency-aware) - # Throughput - total_tokens: int = 0 - total_audio_s: float = 0.0 - mean_tokens_per_request: float = 0.0 - token_throughput: float = 0.0 - request_throughput: float = 0.0 - per_request: list = field(default_factory=list) - - # --------------------------------------------------------------------------- # Inference # --------------------------------------------------------------------------- def _extract_request_output(stage_output): - """Return the RequestOutput-like object from a yielded stage output. - - AsyncOmni stages may yield either a wrapper carrying ``.request_output`` - (qwen3-tts style) or the RequestOutput directly (easymagpie demo style). - """ return getattr(stage_output, "request_output", stage_output) -def _new_audio_frames(payload, prev_num_tokens: int, cur_num_tokens: int): - """Return the decode frames new since the previous step as a ``(k, C*S)`` tensor. +def _new_audio_frames(payload, prev: int, cur: int): + """New decode frames since the previous step as a ``(k, C*S)`` tensor (or None). - Under ``RequestOutputKind.DELTA`` the engine exposes ``audio_codes`` as a - *growing list* during decode — ``[prefill_prefix, frame_0, frame_1, ...]`` — - where element 0 is the prefill prefix and element ``1 + d`` is decode frame - ``d`` (each a ``(1, C*S)`` tensor). The new frames are therefore the list - elements past the previously consumed count. The prefill (first) and final - (consolidated) steps yield a cumulative *tensor* instead, whose newest - ``n_new`` rows are the new frames. Returns ``None`` when there is nothing new. + DELTA mode exposes ``audio_codes`` as a growing list ``[prefill, frame_0, ...]`` + during decode; prefill/final steps yield a cumulative tensor instead. """ import torch - n_new = cur_num_tokens - prev_num_tokens - if n_new <= 0: + if cur - prev <= 0: return None if isinstance(payload, list): - chunks = [c for c in payload[1 + prev_num_tokens : 1 + cur_num_tokens] if isinstance(c, torch.Tensor)] - chunks = [c for c in chunks if c.numel() > 0] + chunks = [c for c in payload[1 + prev : 1 + cur] if isinstance(c, torch.Tensor) and c.numel() > 0] return torch.cat(chunks, dim=0) if chunks else None - if isinstance(payload, torch.Tensor) and payload.shape[0] >= n_new: - return payload[-n_new:] + if isinstance(payload, torch.Tensor) and payload.shape[0] >= cur - prev: + return payload[-(cur - prev) :] return None +class StepMeter: + """Shared per-request measurement: TTFT, ITL, audio length, audio-EOS.""" + + def __init__(self, meta: ModelMeta): + self.meta = meta + self.result = RequestResult() + self.steps = 0 + self._t_start = time.perf_counter() + self._t_last = None + self._prev_tokens = 0 + self._eos_idx = None + + @property + def eos_reached(self) -> bool: + return self._eos_idx is not None + + def observe(self, stage_output) -> None: + now = time.perf_counter() + ro = _extract_request_output(stage_output) + self.steps += 1 + + cur = self._prev_tokens + if getattr(ro, "outputs", None): + out0 = ro.outputs[0] + cum = getattr(out0, "cumulative_token_ids", None) + cur = len(cum) if cum is not None else len(getattr(out0, "token_ids", []) or []) + if cur <= self._prev_tokens: + return + + if self._t_last is None: + self.result.ttft_s = now - self._t_start + else: + self.result.inter_token_latencies.append(now - self._t_last) + self._t_last = now + + mm = getattr(stage_output, "multimodal_output", None) or {} + new_frames = _new_audio_frames(mm.get("audio_codes"), self._prev_tokens, cur) + if new_frames is not None and new_frames.numel() > 0 and self._eos_idx is None: + for j in range(new_frames.shape[0]): + frame_idx = self._prev_tokens + j + if frame_idx >= self.meta.speech_delay and bool((new_frames[j] == self.meta.audio_eos_id).any()): + self._eos_idx = frame_idx + self.result.eos_reached = True + break + self._prev_tokens = cur + + def finalize(self) -> RequestResult: + e2e_s = time.perf_counter() - self._t_start + self.result.success = True + if self.result.ttft_s == 0.0 and self.steps > 0: + self.result.ttft_s = e2e_s + last_frame = self._eos_idx if self._eos_idx is not None else self._prev_tokens + audio_frames = max(0, last_frame - self.meta.speech_delay) + self.result.audio_s = audio_frames * self.meta.frame_stacking_factor / CODEC_FRAME_RATE + return self.result + + def mark_error(self, exc) -> RequestResult: + self.result.error = str(exc) + return self.result + + async def run_one_request( omni, - prompt: dict, + inputs, sampling_params, request_id: str, meta: ModelMeta, - codec_fps: float, - stop_on_eos: bool, + pace=None, + max_steps: Optional[int] = None, ) -> RequestResult: - """Submit one TTS request, collect per-token timing and audio length. - - Each engine step yields one decoded frame (one layer-0 token). We time the - first token (TTFT) and the gaps between subsequent tokens (ITL). - - **Delta consumption.** The request runs in ``RequestOutputKind.DELTA``, so the - engine exposes ``multimodal_output["audio_codes"]`` as a *growing list* during - decode — ``[prefill_prefix, frame_0, frame_1, ...]`` — appending one - ``(1, C*S)`` frame per step instead of re-sending the whole cumulative tensor. - We consume only the frames new since the previous step (see - :func:`_new_audio_frames`). The audio EOS lives in the codebooks (not the vLLM - token stream), so we scan only those delta frames and stop at the EOS frame to - recover the real synthesized length. - """ - import torch - - result = RequestResult() - t_start = time.perf_counter() - t_last_token = None - prev_num_tokens = 0 - eos_decode_idx = None # 0-based decode-frame index where audio EOS appears - delta_frames_total = 0 # decode frames consumed as per-step deltas (sanity check) + """Drain one request's per-step outputs and measure it via :class:`StepMeter`. + ``inputs`` is the prompt dict (whole-text) or an async generator of + ``StreamingInput`` chunks; ``pace`` (streaming only) is awaited after each + frame to release the next chunk. Stops at audio-EOS / backbone stop token. + """ + meter = StepMeter(meta) + gen = None try: - gen = omni.generate( - prompt, - sampling_params_list=[sampling_params], - request_id=request_id, - ) + gen = omni.generate(inputs, sampling_params_list=[sampling_params], request_id=request_id) async for stage_output in gen: - now = time.perf_counter() + meter.observe(stage_output) ro = _extract_request_output(stage_output) - result.steps += 1 - - cur_num_tokens = prev_num_tokens - if hasattr(ro, "outputs") and ro.outputs: - out0 = ro.outputs[0] - cum_ids = getattr(out0, "cumulative_token_ids", None) - if cum_ids is not None: - cur_num_tokens = len(cum_ids) - else: - cur_num_tokens = len(getattr(out0, "token_ids", []) or []) - - if cur_num_tokens > prev_num_tokens: - if t_last_token is None: - result.ttft_s = now - t_start - else: - result.inter_token_latencies.append(now - t_last_token) - t_last_token = now - - # ── Delta: process only the audio frames produced *this step* ── - # DELTA mode appends one (1, C*S) frame per step to the audio_codes - # list, so we pull just the frames new since the previous step. - mm = getattr(stage_output, "multimodal_output", None) or {} - new_frames = _new_audio_frames(mm.get("audio_codes"), prev_num_tokens, cur_num_tokens) - if new_frames is not None and new_frames.numel() > 0: - delta_frames_total += int(new_frames.shape[0]) - # Audio-EOS detection scans only the delta frames. EOS in ANY - # codebook (not just codebook 0) — mirrors the reference EOS - # check and the model's own stop signal. Only meaningful past - # the speech delay. - if eos_decode_idx is None: - for j in range(new_frames.shape[0]): - frame_idx = prev_num_tokens + j # 0-based decode-frame index - if frame_idx < meta.speech_delay: - continue - if bool((new_frames[j] == meta.audio_eos_id).any()): - eos_decode_idx = frame_idx - result.eos_reached = True - break - - prev_num_tokens = cur_num_tokens - - if eos_decode_idx is not None and stop_on_eos: - break + co = ro.outputs[0] if getattr(ro, "outputs", None) else None + if meter.eos_reached or getattr(co, "stop_reason", None) == meta.stop_token_id: + break + if max_steps is not None and meter.steps >= max_steps: + break + if pace is not None: + await pace() + meter.finalize() + except Exception as exc: + meter.mark_error(exc) + logger.error("Request %s failed: %s", request_id, exc) + finally: + if gen is not None: + try: + await gen.aclose() + except Exception: + pass + return meter.result - t_end = time.perf_counter() - result.e2e_s = t_end - t_start - result.num_generated = prev_num_tokens - result.delta_frames = delta_frames_total - result.success = True - if result.ttft_s == 0.0 and result.steps > 0: - result.ttft_s = t_end - t_start +def _clone_sampling_params(sampling_params, max_tokens: int): + import copy - # Real audio length: frames between the start of speech (speech_delay) - # and the EOS frame (or the full decode if no EOS was emitted). - last_audio_frame = eos_decode_idx if eos_decode_idx is not None else prev_num_tokens - result.audio_frames = max(0, last_audio_frame - meta.speech_delay) - if codec_fps > 0: - result.audio_s = result.audio_frames * meta.frame_stacking_factor / codec_fps - result.rtx = result.audio_s / result.e2e_s if result.e2e_s > 0 else 0.0 + sp = copy.deepcopy(sampling_params) + sp.max_tokens = max(1, int(max_tokens)) + return sp - except Exception as exc: - result.e2e_s = time.perf_counter() - t_start - result.error = str(exc) - logger.error("Request %s failed: %s", request_id, exc) - finally: - # Make sure the async generator is closed (aborts the request in the - # engine when we broke out early on EOS). - try: - await gen.aclose() - except Exception: - pass - return result +def build_streaming_request(text: str, meta: ModelMeta, stream_params, max_new_tokens: int): + """Async ``StreamingInput`` feed + pacing coroutine (§5 of the demo). + + Prefill (speaker + context, no text), one chunk per subword with + ``max_tokens=1``, then a ``-1`` mask sentinel with a larger tail budget so the + model free-runs to audio-EOS. Input is paced by output via the queue. + """ + try: + from vllm.engine.protocol import StreamingInput + except ImportError: + from vllm.v1.engine.async_llm import StreamingInput + + prefill_info = { + "speaker_embedding": meta.speaker_embedding, + "context_text": CONTEXT_TEXT, + "temperature": LT_TEMPERATURE, + "top_k": LT_TOPK, + } + text_ids = list(meta.tokenizer.encode(text, add_special_tokens=False)) + [meta.text_eos_id] + tail_params = _clone_sampling_params(stream_params, max_new_tokens - len(text_ids)) + go_queue: asyncio.Queue = asyncio.Queue() + + async def inputs(): + yield StreamingInput( + prompt={"prompt_token_ids": [0] * meta.prompt_len, "additional_information": prefill_info}, + sampling_params=stream_params, + ) + for tok in text_ids: + await go_queue.get() + yield StreamingInput( + prompt={"prompt_token_ids": [0], "additional_information": {"text_token": int(tok)}}, + sampling_params=stream_params, + ) + await go_queue.get() + yield StreamingInput( + prompt={"prompt_token_ids": [0], "additional_information": {"text_token": -1}}, + sampling_params=tail_params, + ) + + async def pace(): + await go_queue.put(True) + + return inputs(), pace # --------------------------------------------------------------------------- @@ -560,17 +433,14 @@ async def worker( omni, texts: list, meta: ModelMeta, - context_text: str, - lt_temperature: float, - lt_topk: int, sampling_params, - codec_fps: float, - stop_on_eos: bool, + stream_params, + streaming: bool, + max_new_tokens: int, results: list, counter: dict, lock: asyncio.Lock, ): - """Persistent async worker that picks texts until the quota is exhausted.""" while True: async with lock: if counter["remaining"] <= 0: @@ -582,30 +452,17 @@ async def worker( text = texts[idx % len(texts)] request_id = f"bench-easymp-w{worker_id}-{uuid.uuid4().hex[:8]}" - prompt = build_prompt( - text=text, - meta=meta, - context_text=context_text, - lt_temperature=lt_temperature, - lt_topk=lt_topk, - ) - - result = await run_one_request( - omni, - prompt, - sampling_params, - request_id, - meta, - codec_fps, - stop_on_eos, - ) - result.text = text - result.prompt_len = len(prompt["prompt_token_ids"]) + if streaming: + inputs, pace = build_streaming_request(text, meta, stream_params, max_new_tokens) + result = await run_one_request( + omni, inputs, stream_params, request_id, meta, pace=pace, max_steps=4 * max_new_tokens + 16 + ) + else: + result = await run_one_request(omni, build_prompt(text, meta), sampling_params, request_id, meta) async with lock: results.append(result) done = len(results) - if done % 10 == 0 or done == counter["total"]: logger.info(" progress: %d / %d", done, counter["total"]) @@ -615,147 +472,47 @@ async def worker( # --------------------------------------------------------------------------- -def _pct(arr, p): - return float(np.percentile(arr, p)) if len(arr) > 0 else 0.0 - - -def compute_and_print_metrics( - results: list, - duration: float, - concurrency: int, - num_requests: int, -) -> BenchmarkResult: - successful = [r for r in results if r.success] +def compute_and_print_metrics(results: list, duration: float, concurrency: int) -> dict: + ok = [r for r in results if r.success] failed = [r for r in results if not r.success] - bench = BenchmarkResult( - concurrency=concurrency, - num_requests=num_requests, - completed=len(successful), - failed=len(failed), - duration_s=duration, - ) - - if not successful: - print("ERROR: No requests completed successfully.") - return bench - - ttfts = [r.ttft_s * 1000 for r in successful] - e2es = [r.e2e_s * 1000 for r in successful] - rtxs = [r.rtx for r in successful] - all_itls = [] - for r in successful: - all_itls.extend([t * 1000 for t in r.inter_token_latencies]) - gen_tokens = [r.num_generated for r in successful] - - bench.mean_ttft_ms = float(np.mean(ttfts)) - bench.median_ttft_ms = float(np.median(ttfts)) - bench.p95_ttft_ms = _pct(ttfts, 95) - bench.p99_ttft_ms = _pct(ttfts, 99) - - bench.mean_e2e_ms = float(np.mean(e2es)) - bench.median_e2e_ms = float(np.median(e2es)) - bench.p95_e2e_ms = _pct(e2es, 95) - bench.p99_e2e_ms = _pct(e2es, 99) - - if all_itls: - bench.mean_itl_ms = float(np.mean(all_itls)) - bench.median_itl_ms = float(np.median(all_itls)) - bench.p95_itl_ms = _pct(all_itls, 95) - bench.p99_itl_ms = _pct(all_itls, 99) - - bench.mean_rtx = float(np.mean(rtxs)) - bench.median_rtx = float(np.median(rtxs)) - - bench.total_tokens = int(sum(gen_tokens)) - bench.total_audio_s = float(sum(r.audio_s for r in successful)) - bench.mean_tokens_per_request = float(np.mean(gen_tokens)) - bench.token_throughput = bench.total_tokens / duration if duration > 0 else 0.0 - bench.request_throughput = len(successful) / duration if duration > 0 else 0.0 - bench.overall_rtx = bench.total_audio_s / duration if duration > 0 else 0.0 - - bench.per_request = [ - { - "ttft_ms": r.ttft_s * 1000, - "e2e_ms": r.e2e_s * 1000, - "rtx": r.rtx, - "num_generated": r.num_generated, - "delta_frames": r.delta_frames, - "audio_frames": r.audio_frames, - "audio_s": r.audio_s, - "eos_reached": r.eos_reached, - "steps": r.steps, - "prompt_len": r.prompt_len, - "mean_itl_ms": float(np.mean([t * 1000 for t in r.inter_token_latencies])) - if r.inter_token_latencies - else 0.0, - "text": r.text, - } - for r in successful - ] - - eos_hits = sum(1 for r in successful if r.eos_reached) + ttfts = [r.ttft_s * 1000 for r in ok] + itls = [t * 1000 for r in ok for t in r.inter_token_latencies] + total_audio_s = sum(r.audio_s for r in ok) + eos_hits = sum(1 for r in ok if r.eos_reached) + + summary = { + "concurrency": concurrency, + "completed": len(ok), + "failed": len(failed), + "eos_hits": eos_hits, + "duration_s": duration, + "req_per_s": len(ok) / duration if duration > 0 else 0.0, + "ttft_mean_ms": float(np.mean(ttfts)) if ttfts else 0.0, + "ttft_p95_ms": float(np.percentile(ttfts, 95)) if ttfts else 0.0, + "itl_mean_ms": float(np.mean(itls)) if itls else 0.0, + "itl_p95_ms": float(np.percentile(itls, 95)) if itls else 0.0, + "rtf": total_audio_s / duration if duration > 0 else 0.0, + } - W = 56 + W = 48 print(f"\n{'=' * W}") - print(f"{'Benchmark Result':^{W}}") + print(f"{f'Benchmark (concurrency={concurrency})':^{W}}") print(f"{'=' * W}") - print(f"{'Successful requests:':<42}{bench.completed}") - print(f"{'Failed requests:':<42}{bench.failed}") - print(f"{'Reached audio EOS:':<42}{eos_hits} / {bench.completed}") - print(f"{'Concurrency:':<42}{concurrency}") - print(f"{'Wall-clock duration (s):':<42}{duration:.2f}") - print(f"{'Request throughput (req/s):':<42}{bench.request_throughput:.2f}") - - print(f"\n{'-' * W}") - print(f"{'Time to First Token (TTFT)':^{W}}") - print(f"{'-' * W}") - print(f"{'Mean (ms):':<42}{bench.mean_ttft_ms:.2f}") - print(f"{'Median (ms):':<42}{bench.median_ttft_ms:.2f}") - print(f"{'P95 (ms):':<42}{bench.p95_ttft_ms:.2f}") - print(f"{'P99 (ms):':<42}{bench.p99_ttft_ms:.2f}") - - print(f"\n{'-' * W}") - print(f"{'End-to-End Latency (E2E)':^{W}}") - print(f"{'-' * W}") - print(f"{'Mean (ms):':<42}{bench.mean_e2e_ms:.2f}") - print(f"{'Median (ms):':<42}{bench.median_e2e_ms:.2f}") - print(f"{'P95 (ms):':<42}{bench.p95_e2e_ms:.2f}") - print(f"{'P99 (ms):':<42}{bench.p99_e2e_ms:.2f}") - - print(f"\n{'-' * W}") - print(f"{'Inter-Token Latency (ITL)':^{W}}") - print(f"{'-' * W}") - if all_itls: - print(f"{'Mean (ms):':<42}{bench.mean_itl_ms:.2f}") - print(f"{'Median (ms):':<42}{bench.median_itl_ms:.2f}") - print(f"{'P95 (ms):':<42}{bench.p95_itl_ms:.2f}") - print(f"{'P99 (ms):':<42}{bench.p99_itl_ms:.2f}") - else: - print(f"{'(no inter-token data)':^{W}}") - - print(f"\n{'-' * W}") - print(f"{'Real-Time Factor (RTX = audio_s / gen_s)':^{W}}") - print(f"{'-' * W}") - print(f"{'Mean RTX (per request):':<42}{bench.mean_rtx:.2f}x") - print(f"{'Median RTX (per request):':<42}{bench.median_rtx:.2f}x") - print(f"{'Overall RTX (total audio / wall):':<42}{bench.overall_rtx:.2f}x") - - print(f"\n{'-' * W}") - print(f"{'Throughput':^{W}}") - print(f"{'-' * W}") - print(f"{'Total frames generated:':<42}{bench.total_tokens}") - print(f"{'Total audio generated (s):':<42}{bench.total_audio_s:.2f}") - print(f"{'Mean frames / request:':<42}{bench.mean_tokens_per_request:.1f}") - print(f"{'Frame throughput (frames/s):':<42}{bench.token_throughput:.2f}") + if not ok: + print("ERROR: no requests completed successfully.") + if failed: + print(f" e.g. {failed[0].error[:200]}") + return summary + print(f"{'Requests (ok / failed):':<28}{summary['completed']} / {summary['failed']}") + print(f"{'Reached audio EOS:':<28}{eos_hits} / {summary['completed']}") + print(f"{'Duration (s):':<28}{duration:.2f}") + print(f"{'Throughput (req/s):':<28}{summary['req_per_s']:.2f}") + print(f"{'TTFT mean / p95 (ms):':<28}{summary['ttft_mean_ms']:.2f} / {summary['ttft_p95_ms']:.2f}") + print(f"{'ITL mean / p95 (ms):':<28}{summary['itl_mean_ms']:.2f} / {summary['itl_p95_ms']:.2f}") + print(f"{'RTF (audio_s / wall):':<28}{summary['rtf']:.2f}x") print(f"{'=' * W}\n") - - if failed: - print(f" First {min(3, len(failed))} errors:") - for r in failed[:3]: - print(f" {r.error[:200]}") - - return bench + return summary # --------------------------------------------------------------------------- @@ -763,85 +520,64 @@ def compute_and_print_metrics( # --------------------------------------------------------------------------- +async def _run_workers(omni, texts, meta, sampling_params, stream_params, args, n_requests, concurrency): + results: list = [] + counter = {"remaining": n_requests, "issued": 0, "total": n_requests} + lock = asyncio.Lock() + tasks = [ + asyncio.create_task( + worker( + i, omni, texts, meta, sampling_params, stream_params, + args.streaming, args.max_new_tokens, results, counter, lock, + ) + ) + for i in range(concurrency) + ] + await asyncio.gather(*tasks) + return results + + async def main(args): from vllm import SamplingParams from vllm.sampling_params import RequestOutputKind from vllm_omni import AsyncOmni - model_name = args.model - - # ── Load texts ──────────────────────────────────────────────────────── if args.text_file: path = Path(args.text_file) if not path.exists(): print(f"ERROR: text file not found: {path}") return - raw_lines = [line.strip() for line in path.read_text().splitlines() if line.strip()] texts = [] - for line in raw_lines: - if "\t" in line: - texts.append(line.split("\t", 1)[1].strip()) - else: - texts.append(line) + for line in path.read_text().splitlines(): + line = line.strip() + if line: + texts.append(line.split("\t", 1)[1].strip() if "\t" in line else line) texts = [t for t in texts if t] - logger.info("Loaded %d texts from %s", len(texts), path) else: texts = DEFAULT_PROMPTS - logger.info("Using %d default prompts", len(texts)) - if not texts: print("ERROR: no texts available.") return + logger.info("Loaded %d texts", len(texts)) - # ── Read arch scalars + tokenizer + speaker embedding ───────────────── - logger.info("Reading model metadata from %s ...", model_name) - meta = _load_model_meta( - model_dir=model_name, - speaker=args.speaker, - speaker_embedding_path=args.speaker_embedding, - context_text=args.context_text, - ) + meta = _load_model_meta(args.model) logger.info( "prompt_len=%d audio_eos_id=%d speech_delay=%d frame_stacking=%d", - meta.prompt_len, - meta.audio_eos_id, - meta.speech_delay, - meta.frame_stacking_factor, + meta.prompt_len, meta.audio_eos_id, meta.speech_delay, meta.frame_stacking_factor, ) if meta.prompt_len + args.max_new_tokens > args.max_model_len: - logger.warning( - "prompt_len (%d) + max_new_tokens (%d) exceeds max_model_len (%d); raise --max-model-len.", - meta.prompt_len, - args.max_new_tokens, - args.max_model_len, - ) - - max_concurrency = max(args.concurrency) + logger.warning("prompt_len + max_new_tokens exceeds max_model_len (%d)", args.max_model_len) + logger.info("Mode: %s", "streaming-text" if args.streaming else "whole-text") - # ── Build stage config ──────────────────────────────────────────────── - stage_cfg = _build_easymagpie_stage_config( - max_num_seqs=max_concurrency, - profile=args.profile, - torch_profiler_dir=args.torch_profiler_dir, - with_stack=args.with_stack, - record_shapes=args.record_shapes, - gpu_memory_utilization=args.gpu_memory_utilization, + stage_cfg = _build_stage_config( + max_num_seqs=max(args.concurrency), max_model_len=args.max_model_len, max_num_batched_tokens=args.max_num_batched_tokens, - enforce_eager=args.enforce_eager, max_new_tokens=args.max_new_tokens, - dtype=args.dtype, - distributed_executor_backend=args.distributed_executor_backend, - cudagraph_mode=args.cudagraph_mode, + profile=args.profile, + torch_profiler_dir=args.torch_profiler_dir, load_format=args.load_format, ) - if args.cudagraph_mode is not None and args.enforce_eager: - logger.warning( - "--cudagraph-mode %s is ignored because --enforce-eager disables CUDA graphs.", - args.cudagraph_mode, - ) - elif args.cudagraph_mode is not None: - logger.info("CUDA-graph mode: %s", args.cudagraph_mode) tmp_config_path = _write_temp_stage_config(stage_cfg) sampling_params = SamplingParams( @@ -849,155 +585,68 @@ async def main(args): max_tokens=args.max_new_tokens, detokenize=False, ignore_eos=True, - # The model emits this backbone token at the audio-EOS frame (audio EOS in - # any codebook), so vLLM stops the request there instead of decoding the - # full budget. stop_token_ids is honored even with ignore_eos. stop_token_ids=[meta.stop_token_id], - # DELTA: surface audio_codes as a *growing list* of per-step frame deltas - # during decode (element 0 = prefill prefix, element 1+d = decode frame d) - # instead of re-sending the whole cumulative tensor every step. This is the - # same streaming-consumption pattern the Triton deployment uses. + output_kind=RequestOutputKind.DELTA, + ) + # Streaming: max_tokens=1 -> one chunk per decoded frame. + stream_params = SamplingParams( + temperature=0.0, + max_tokens=1, + detokenize=False, + ignore_eos=True, + stop_token_ids=[meta.stop_token_id], output_kind=RequestOutputKind.DELTA, ) try: - logger.info("Creating AsyncOmni engine (EasyMagpie talker only) for %s ...", model_name) + logger.info("Creating AsyncOmni engine for %s ...", args.model) omni = AsyncOmni( - model=model_name, + model=args.model, stage_configs_path=tmp_config_path, - log_stats=args.log_stats, - stage_init_timeout=args.stage_init_timeout, + log_stats=False, + stage_init_timeout=STAGE_INIT_TIMEOUT, ) - logger.info("Engine ready (single stage: EasyMagpie talker).") - - all_bench_results = [] + logger.info("Engine ready.") + summaries = [] for concurrency in args.concurrency: - logger.info( - "=== concurrency=%d requests=%d ===", - concurrency, - args.num_requests, - ) + logger.info("=== concurrency=%d requests=%d ===", concurrency, args.num_requests) - # ── Warmup ──────────────────────────────────────────────────── warmup_count = 0 if args.no_warmup else args.num_warmups * concurrency if warmup_count > 0: - logger.info("Warming up with %d requests (concurrency=%d)...", warmup_count, concurrency) - warmup_results: list = [] - warmup_counter = { - "remaining": warmup_count, - "issued": 0, - "total": warmup_count, - } - warmup_lock = asyncio.Lock() - warmup_tasks = [ - asyncio.create_task( - worker( - worker_id=i, - omni=omni, - texts=texts, - meta=meta, - context_text=args.context_text, - lt_temperature=args.lt_temperature, - lt_topk=args.lt_topk, - sampling_params=sampling_params, - codec_fps=args.codec_frame_rate, - stop_on_eos=not args.no_stop_on_eos, - results=warmup_results, - counter=warmup_counter, - lock=warmup_lock, - ) - ) - for i in range(concurrency) - ] - await asyncio.gather(*warmup_tasks) - warmup_ok = sum(1 for r in warmup_results if r.success) - logger.info("Warmup done: %d / %d succeeded.", warmup_ok, warmup_count) - - # ── Benchmark run ───────────────────────────────────────────── - logger.info("Starting benchmark run (%d requests, concurrency=%d)...", args.num_requests, concurrency) - - bench_results: list = [] - counter = { - "remaining": args.num_requests, - "issued": 0, - "total": args.num_requests, - } - lock = asyncio.Lock() + logger.info("Warming up with %d requests...", warmup_count) + await _run_workers(omni, texts, meta, sampling_params, stream_params, args, warmup_count, concurrency) if args.profile: - logger.info("Starting profiler ...") - await omni.start_profile( - profile_prefix=args.profile_prefix, - stages=[0], - ) - - start_time = time.perf_counter() + await omni.start_profile(stages=[0]) + start = time.perf_counter() try: - tasks = [ - asyncio.create_task( - worker( - worker_id=i, - omni=omni, - texts=texts, - meta=meta, - context_text=args.context_text, - lt_temperature=args.lt_temperature, - lt_topk=args.lt_topk, - sampling_params=sampling_params, - codec_fps=args.codec_frame_rate, - stop_on_eos=not args.no_stop_on_eos, - results=bench_results, - counter=counter, - lock=lock, - ) - ) - for i in range(concurrency) - ] - await asyncio.gather(*tasks) + results = await _run_workers( + omni, texts, meta, sampling_params, stream_params, args, args.num_requests, concurrency + ) finally: if args.profile: - logger.info("Stopping profiler ...") await omni.stop_profile(stages=[0]) + duration = time.perf_counter() - start - duration = time.perf_counter() - start_time - - bench = compute_and_print_metrics( - bench_results, - duration, - concurrency, - args.num_requests, - ) - bench.config_name = args.config_name - all_bench_results.append(asdict(bench)) + summaries.append(compute_and_print_metrics(results, duration, concurrency)) - # ── Top-level summary (one line per concurrency level) ───────────── print(f"\n{'=' * 56}") - print(f"{'Summary (' + args.config_name + ')':^56}") + print(f"{'Summary':^56}") print(f"{'=' * 56}") - for r in all_bench_results: + for s in summaries: print( - f"concurrency={r['concurrency']}: " - f"req/s {r['request_throughput']:.2f}, " - f"ttft {r['mean_ttft_ms']:.2f}ms, " - f"itl {r['mean_itl_ms']:.2f}ms" + f"concurrency={s['concurrency']}: " + f"req/s {s['req_per_s']:.2f}, " + f"ttft {s['ttft_mean_ms']:.1f}ms, " + f"itl {s['itl_mean_ms']:.1f}ms, " + f"rtf {s['rtf']:.2f}x" ) print(f"{'=' * 56}\n") - # ── Save results ────────────────────────────────────────────────── - if args.result_dir: - result_dir = Path(args.result_dir) - result_dir.mkdir(parents=True, exist_ok=True) - timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") - result_file = result_dir / f"bench_easymagpie_{args.config_name}_{timestamp}.json" - with open(result_file, "w") as f: - json.dump(all_bench_results, f, indent=2) - logger.info("Results saved to %s", result_file) - omni.shutdown() finally: os.unlink(tmp_config_path) - logger.info("Done.") @@ -1007,156 +656,25 @@ def parse_args(): formatter_class=argparse.RawDescriptionHelpFormatter, epilog=__doc__, ) - - model = parser.add_argument_group("model / input") - model.add_argument( - "--model", - type=str, - default="./easymp_vllm_model", - help="Converted EasyMagpie model directory (output of easy_magpietts_convert_to_vllm.py)", - ) - model.add_argument( - "--text-file", - type=str, - default=None, - help="Path to text file (one utterance per line, optionally tab-separated with text in 2nd column)", - ) - model.add_argument( - "--speaker", - type=str, - default="eng", - help="Speaker embedding name under /speaker_embeddings/.pt", - ) - model.add_argument( - "--speaker-embedding", - type=str, - default=None, - help="Explicit path to a speaker embedding .pt (overrides --speaker)", - ) - model.add_argument( - "--context-text", - type=str, - default="[EN]", - help="Conditioning string tokenized + embedded in-engine (e.g. '[EN]')", - ) - model.add_argument( - "--lt-temperature", - type=float, - default=0.0, - help="Audio (local-transformer) sampling temperature (0.0 == argmax)", - ) - model.add_argument( - "--lt-topk", - type=int, - default=80, - help="Audio (local-transformer) sampling top-k", - ) - model.add_argument( - "--max-new-tokens", - type=int, - default=256, - help="Max decode frames per request (decode budget; trimmed at audio EOS)", - ) - model.add_argument( - "--codec-frame-rate", - type=float, - default=25.0, - help="Codec frame rate (Hz) used to convert decoded frames to audio seconds " - "(default 25 for the 25fps spectral codec)", - ) - - bench = parser.add_argument_group("benchmark") - bench.add_argument( - "-c", - "--concurrency", - type=int, - nargs="+", - default=[1], - help="Concurrency levels to test (space-separated, default: 1)", - ) - bench.add_argument( - "-n", - "--num-requests", - type=int, - default=50, - help="Total number of requests per concurrency level (default: 50)", - ) - bench.add_argument( - "--num-warmups", - type=int, - default=3, - help="Warmup rounds per concurrency level (total warmup = concurrency * this, default: 3)", - ) - bench.add_argument("--no-warmup", action="store_true", help="Skip warmup") - bench.add_argument( - "--no-stop-on-eos", - action="store_true", - help="Do not stop at the audio-EOS frame; run the full decode budget every request", - ) - bench.add_argument( - "--config-name", - type=str, - default="easymagpie", - help="Label for this run (used in result filenames)", - ) - bench.add_argument( - "--result-dir", - type=str, - default=None, - help="Directory to save JSON results", - ) - - engine = parser.add_argument_group("engine") - engine.add_argument( + parser.add_argument("--model", type=str, default="./easymp_vllm_model", help="Converted EasyMagpie model dir") + parser.add_argument("--text-file", type=str, default=None, help="One utterance per line (optionally tab-sep)") + parser.add_argument("--streaming", action="store_true", help="Benchmark the token-streamed input path") + parser.add_argument("-c", "--concurrency", type=int, nargs="+", default=[1], help="Concurrency levels to test") + parser.add_argument("-n", "--num-requests", type=int, default=50, help="Requests per concurrency level") + parser.add_argument("--num-warmups", type=int, default=3, help="Warmup rounds (total = concurrency * this)") + parser.add_argument("--no-warmup", action="store_true", help="Skip warmup") + parser.add_argument("--max-new-tokens", type=int, default=256, help="Max decode frames per request") + parser.add_argument("--max-model-len", type=int, default=1024) + parser.add_argument("--max-num-batched-tokens", type=int, default=1024) + parser.add_argument( "--load-format", type=str, default=None, choices=["auto", "dummy", "safetensors", "pt"], - help="Weight loading strategy. Use 'dummy' to initialize random weights and skip the " - "checkpoint (pair with a dummy config dir to benchmark a backbone without a trained " - "checkpoint). Default: unset (vLLM default 'auto').", + help="Weight loading strategy ('dummy' = random weights, skip checkpoint)", ) - engine.add_argument("--gpu-memory-utilization", type=float, default=0.5) - engine.add_argument("--max-model-len", type=int, default=1024) - engine.add_argument("--max-num-batched-tokens", type=int, default=1024) - engine.add_argument("--dtype", type=str, default="float16", help="Model dtype (float16 / bfloat16)") - engine.add_argument("--enforce-eager", action="store_true") - engine.add_argument( - "--cudagraph-mode", - type=str, - default=None, - choices=["NONE", "PIECEWISE", "FULL", "FULL_DECODE_ONLY", "FULL_AND_PIECEWISE"], - help="vLLM CUDA-graph capture strategy (compilation_config.cudagraph_mode). " - "Default: unset (vLLM default, FULL_AND_PIECEWISE). Use PIECEWISE to capture the " - "backbone and local transformer as separate graphs during decode so their split is " - "visible in a profiler (slightly slower than the default full decode graph). " - "Ignored when --enforce-eager is set.", - ) - engine.add_argument("--stage-init-timeout", type=int, default=300) - engine.add_argument("--log-stats", action="store_true", default=False) - engine.add_argument( - "--distributed-executor-backend", - type=str, - default="uni", - choices=["uni", "mp", "ray"], - help="vLLM executor backend. 'uni' runs the worker in-process and " - "avoids shm_broadcast IPC round-trips (recommended for TP=1, single " - "GPU). Default: uni.", - ) - - prof = parser.add_argument_group("profiling") - prof.add_argument( - "--profile", - action="store_true", - help="Enable torch profiler during the benchmark run", - ) - prof.add_argument("--profile-prefix", type=str, default=None, help="Prefix for profiler trace filenames") - prof.add_argument( - "--torch-profiler-dir", type=str, default="./profiler_traces", help="Directory for torch profiler traces" - ) - prof.add_argument("--with-stack", action="store_true", help="Record Python call stacks in profiler") - prof.add_argument("--record-shapes", action="store_true", help="Record tensor shapes in profiler") - + parser.add_argument("--profile", action="store_true", help="Enable torch profiler (with stack + shapes)") + parser.add_argument("--torch-profiler-dir", type=str, default="./profiler_traces", help="Profiler trace dir") return parser.parse_args() From 2b19ee51016b6c6146367483af6b1414d696a14d Mon Sep 17 00:00:00 2001 From: Viacheslav Klimkov Date: Wed, 10 Jun 2026 12:01:25 +0200 Subject: [PATCH 32/45] examples/tts/easymagpie_vllm_omni: add input streaming support into tts service Signed-off-by: Viacheslav Klimkov --- .../easymagpie_vllm_omni/benchmark_service.py | 183 +++---- .../benchmark_streaming_service.py | 456 ++++++++++++++++++ .../model_repository/easymp/1/model.py | 443 ++++++++++++++--- .../model_repository/easymp/config.pbtxt | 33 ++ .../run_service_request.ipynb | 153 +++++- 5 files changed, 1106 insertions(+), 162 deletions(-) create mode 100644 examples/tts/easymagpie_vllm_omni/benchmark_streaming_service.py diff --git a/examples/tts/easymagpie_vllm_omni/benchmark_service.py b/examples/tts/easymagpie_vllm_omni/benchmark_service.py index ac002b853495..f404ed9ecdd4 100644 --- a/examples/tts/easymagpie_vllm_omni/benchmark_service.py +++ b/examples/tts/easymagpie_vllm_omni/benchmark_service.py @@ -12,18 +12,15 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -""" -Benchmark script for the EasyMagpie TTS Triton server (decoupled mode, gRPC). +"""Benchmark the EasyMagpie TTS Triton server (decoupled mode, gRPC, whole-text). -Spawns N concurrent workers that send TTS requests in parallel against the -``easymp`` Triton model (see ``model_repository/easymp/config.pbtxt``). -Each line of the text file is parsed as ``\\t``. -Texts are randomly sampled for each request. +Spawns N concurrent workers that send TTS requests against the ``easymp`` Triton +model. Each line of the text file is ``\\t``; texts are sampled at +random per request. Multiple concurrency levels can be benchmarked in sequence. Usage: - python benchmark_easymagpie_triton.py --text-file vctk_subset.txt --num-requests 100 --num-workers 8 - python benchmark_easymagpie_triton.py --text-file vctk_subset.txt --num-requests 50 \ - --output-dir out_wavs + python benchmark_service.py --text-file vctk_subset.txt -n 100 -c 8 + python benchmark_service.py --text-file vctk_subset.txt -n 50 -c 1 4 8 """ import argparse @@ -38,7 +35,7 @@ import numpy as np import tritonclient.grpc as grpcclient -SAMPLE_RATE = 22_050 # codec output_sample_rate (matches run_server_request.ipynb) +SAMPLE_RATE = 22_050 MODEL_NAME = "easymp" @@ -77,10 +74,6 @@ def synthesize( text: str, chunk_timeout: float, ): - """Send one TTS request and collect streamed chunks. - - Returns ``(audio, ttfa_s, elapsed_s, error)``. - """ text_input = grpcclient.InferInput("text", [1, 1], "BYTES") text_input.set_data_from_numpy(np.array([[text]], dtype=object)) @@ -131,6 +124,7 @@ def worker( stats: BenchmarkStats, chunk_timeout: float, output_dir: Path | None, + verbose: bool, ): result_q: queue.Queue = queue.Queue() client = grpcclient.InferenceServerClient(url=triton_url) @@ -147,39 +141,23 @@ def worker( audio, ttfa, elapsed, error = synthesize(client, result_q, text, chunk_timeout) if error is not None: - # Reset the stream so late chunks don't bleed into the next - # request. client.stop_stream() client.start_stream(callback=lambda result, error: result_q.put((result, error))) - stats.add( - RequestResult( - uttid=uttid, - num_samples=0, - duration_s=elapsed, - ttfa_s=ttfa, - error=error, - ) - ) - print(f"[worker {worker_id:02d}] request {task_idx} ({uttid}) FAILED ({elapsed:.1f}s) — {error}") + stats.add(RequestResult(uttid=uttid, num_samples=0, duration_s=elapsed, ttfa_s=ttfa, error=error)) + if verbose: + print(f"[worker {worker_id:02d}] req {task_idx} ({uttid}) FAILED ({elapsed:.1f}s) — {error}") continue num_samples = len(audio) if output_dir is not None and num_samples > 0: _save_wav(output_dir / f"{uttid}.wav", audio) - stats.add( - RequestResult( - uttid=uttid, - num_samples=num_samples, - duration_s=elapsed, - ttfa_s=ttfa, + stats.add(RequestResult(uttid=uttid, num_samples=num_samples, duration_s=elapsed, ttfa_s=ttfa)) + if verbose: + print( + f"[worker {worker_id:02d}] req {task_idx} ({uttid}) — " + f"{num_samples / SAMPLE_RATE:.2f}s audio in {elapsed:.2f}s (TTFA {ttfa * 1000:.0f}ms)" ) - ) - print( - f"[worker {worker_id:02d}] request {task_idx} ({uttid}) done — " - f"{num_samples / SAMPLE_RATE:.2f}s audio in {elapsed:.2f}s " - f"(TTFA: {ttfa:.3f}s)" - ) finally: client.stop_stream() @@ -208,6 +186,7 @@ def _run_workers( num_tasks: int, chunk_timeout: float, output_dir: Path | None, + verbose: bool, ) -> tuple[BenchmarkStats, float]: task_queue = list(range(num_tasks)) queue_lock = threading.Lock() @@ -216,7 +195,7 @@ def _run_workers( threads = [ threading.Thread( target=worker, - args=(i, triton_url, items, task_queue, queue_lock, stats, chunk_timeout, output_dir), + args=(i, triton_url, items, task_queue, queue_lock, stats, chunk_timeout, output_dir, verbose), ) for i in range(num_workers) ] @@ -228,21 +207,51 @@ def _run_workers( return stats, time.perf_counter() - wall_start +def _percentile(sorted_vals: list[float], pct: float) -> float: + if not sorted_vals: + return 0.0 + idx = min(len(sorted_vals) - 1, int(len(sorted_vals) * pct)) + return sorted_vals[idx] + + +def _summarize(stats: BenchmarkStats, wall_s: float, concurrency: int) -> dict: + successes = [r for r in stats.results if r.error is None] + failures = [r for r in stats.results if r.error is not None] + audio_s = sum(r.num_samples for r in successes) / SAMPLE_RATE + ttfas_ms = sorted(r.ttfa_s * 1000 for r in successes) + return { + "concurrency": concurrency, + "failed": len(failures), + "wall_s": wall_s, + "audio_s": audio_s, + "rtx": audio_s / wall_s if wall_s > 0 else 0.0, + "tput": len(successes) / wall_s if wall_s > 0 else 0.0, + "ttfa_mean_ms": (sum(ttfas_ms) / len(ttfas_ms)) if ttfas_ms else 0.0, + "ttfa_p95_ms": _percentile(ttfas_ms, 0.95), + } + + +def _print_summary(s: dict): + print( + f"[concurrency={s['concurrency']}] rtx = synt / time = " + f"{s['rtx']:.2f}x = {s['audio_s']:.0f} / {s['wall_s']:.2f}" + ) + print( + f"throughput={s['tput']:.2f} req/s; failed = {s['failed']}; " + f"TTFA={s['ttfa_mean_ms']:.1f} / {s['ttfa_p95_ms']:.1f} (p95)" + ) + + def main(): parser = argparse.ArgumentParser(description="Benchmark EasyMagpie TTS Triton server") parser.add_argument("--text-file", required=True, help="Path to file with '\\t' per line") - parser.add_argument("--num-requests", type=int, required=True, help="Total number of requests to send") - parser.add_argument("--num-workers", type=int, default=4, help="Number of concurrent workers (default: 4)") - parser.add_argument( - "--triton-url", default="localhost:8001", help="Triton gRPC endpoint (default: localhost:8001)" - ) + parser.add_argument("-n", "--num-requests", type=int, required=True, help="Requests per concurrency level") + parser.add_argument("-c", "--concurrency", type=int, nargs="+", default=[4], help="Concurrency levels to test") + parser.add_argument("--triton-url", default="localhost:8001", help="Triton gRPC endpoint (default localhost:8001)") parser.add_argument("--no-warmup", action="store_true", help="Skip warmup phase (3 requests per worker)") - parser.add_argument( - "--chunk-timeout", type=float, default=60, help="Per-chunk receive timeout in seconds (default: 60)" - ) - parser.add_argument( - "--output-dir", default=None, help="If set, write each generated waveform to /.wav" - ) + parser.add_argument("--chunk-timeout", type=float, default=60, help="Per-chunk receive timeout, s (default: 60)") + parser.add_argument("--output-dir", default=None, help="If set, write each waveform to /.wav") + parser.add_argument("--verbose", action="store_true", help="Print per-request lines") args = parser.parse_args() items = _load_items(args.text_file) @@ -255,62 +264,26 @@ def main(): output_dir = Path(args.output_dir) output_dir.mkdir(parents=True, exist_ok=True) - print(f"Loaded {len(items)} utterances from {args.text_file}") - print(f"Sending {args.num_requests} requests with {args.num_workers} workers to {args.triton_url}") - if output_dir is not None: - print(f"Writing WAVs to {output_dir.resolve()}") - print("-" * 70) - - if not args.no_warmup: - total_warmup = args.num_workers * 3 - print(f"Warmup: {total_warmup} requests (3 per worker) ...") - _run_workers( - args.num_workers, - args.triton_url, - items, - total_warmup, - args.chunk_timeout, - output_dir=None, - ) - print("Warmup complete.") - print("-" * 70) - - stats, wall_elapsed = _run_workers( - args.num_workers, - args.triton_url, - items, - args.num_requests, - args.chunk_timeout, - output_dir, - ) + print(f"Loaded {len(items)} utterances; {args.num_requests} req/level; concurrency {args.concurrency}") - successes = [r for r in stats.results if r.error is None] - failures = [r for r in stats.results if r.error is not None] - total_audio_seconds = sum(r.num_samples for r in successes) / SAMPLE_RATE - - print() - print("=" * 70) - print("BENCHMARK RESULTS") - print("=" * 70) - print(f" Total requests sent: {args.num_requests}") - print(f" Successful: {len(successes)}") - print(f" Failed: {len(failures)}") - print(f" Concurrent workers: {args.num_workers}") - print() - print(f" Wall-clock time: {wall_elapsed:.2f} s") - print(f" Total audio synthesized: {total_audio_seconds:.2f} s") - print(f" Real-time factor (RTF): {total_audio_seconds / wall_elapsed:.2f}x") - print(f" Throughput: {len(successes) / wall_elapsed:.2f} requests/s") - - if successes: - ttfas_ms = sorted(r.ttfa_s * 1000 for r in successes) - mean_ttfa = sum(ttfas_ms) / len(ttfas_ms) - print() - print(" Time to first audio (TTFA):") - print(f" mean: {mean_ttfa:.1f} ms") - print(f" p95: {ttfas_ms[int(len(ttfas_ms) * 0.95)]:.1f} ms") - - print("=" * 70) + summaries = [] + for concurrency in args.concurrency: + if not args.no_warmup: + _run_workers( + concurrency, args.triton_url, items, concurrency * 3, args.chunk_timeout, None, args.verbose + ) + + stats, wall_elapsed = _run_workers( + concurrency, args.triton_url, items, args.num_requests, args.chunk_timeout, output_dir, args.verbose + ) + summary = _summarize(stats, wall_elapsed, concurrency) + summaries.append(summary) + _print_summary(summary) + + if len(summaries) > 1: + print("\n=== Summary ===") + for s in summaries: + _print_summary(s) if __name__ == "__main__": diff --git a/examples/tts/easymagpie_vllm_omni/benchmark_streaming_service.py b/examples/tts/easymagpie_vllm_omni/benchmark_streaming_service.py new file mode 100644 index 000000000000..2f65c40881b3 --- /dev/null +++ b/examples/tts/easymagpie_vllm_omni/benchmark_streaming_service.py @@ -0,0 +1,456 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Benchmark the EasyMagpie TTS Triton server in streaming-text mode (gRPC). + +Unlike ``benchmark_service.py`` (whole-text), each request feeds subword ids one +chunk at a time over a single gRPC stream (``stream_start`` ... ``stream_end``); +all audio rides back on the ``stream_start`` request's response. Multiple +concurrency levels can be benchmarked in sequence. + +Usage: + python benchmark_streaming_service.py --text-file vctk_subset.txt \ + --model-dir ./easymp_vllm_model -n 100 -c 8 + python benchmark_streaming_service.py --text-file vctk_subset.txt \ + --model-dir ./easymp_vllm_model -n 50 -c 1 4 8 +""" + +import argparse +import queue +import random +import threading +import time +import uuid +import wave +from dataclasses import dataclass, field +from pathlib import Path + +import numpy as np +import tritonclient.grpc as grpcclient +from transformers import AutoTokenizer + +SAMPLE_RATE = 22_050 +MODEL_NAME = "easymp" + + +@dataclass +class RequestResult: + uttid: str + num_samples: int = 0 + elapsed_s: float = 0.0 + ttfa_s: float = 0.0 + reached_eos: bool = False + keepup_ratios: list[float] = field(default_factory=list) + audio: np.ndarray | None = None + error: str | None = None + + +@dataclass +class BenchmarkStats: + lock: threading.Lock = field(default_factory=threading.Lock) + results: list[RequestResult] = field(default_factory=list) + + def add(self, result: RequestResult): + with self.lock: + self.results.append(result) + + +def _str_input(name: str, value: str): + t = grpcclient.InferInput(name, [1, 1], "BYTES") + t.set_data_from_numpy(np.array([[value]], dtype=object)) + return t + + +def _bool_input(name: str, value: bool): + t = grpcclient.InferInput(name, [1, 1], "BOOL") + t.set_data_from_numpy(np.array([[value]], dtype=bool)) + return t + + +def _token_input(tokens): + t = grpcclient.InferInput("text_token", [1, len(tokens)], "INT32") + t.set_data_from_numpy(np.array([tokens], dtype=np.int32)) + return t + + +def _save_wav(path: Path, audio: np.ndarray, sample_rate: int = SAMPLE_RATE): + audio = np.clip(audio, -1.0, 1.0) + pcm = (audio * 32767.0).astype(np.int16) + with wave.open(str(path), "wb") as wf: + wf.setnchannels(1) + wf.setsampwidth(2) + wf.setframerate(sample_rate) + wf.writeframes(pcm.tobytes()) + + +def synthesize_streaming( + client: grpcclient.InferenceServerClient, + result_q: queue.Queue, + tokenizer, + uttid: str, + text: str, + speaker: str, + context_text: str, + tokens_per_chunk: int, + token_delay: float, + chunk_timeout: float, + save_audio: bool = False, +) -> RequestResult: + token_ids = tokenizer.encode(text, add_special_tokens=False) + chunks = [token_ids[i : i + tokens_per_chunk] for i in range(0, len(token_ids), tokens_per_chunk)] or [[]] + num_requests = len(chunks) + + stream_id = uuid.uuid4().hex + start_rid = f"{stream_id}-start" + + def _send(inputs, request_id): + client.async_stream_infer( + model_name=MODEL_NAME, + inputs=inputs, + outputs=[grpcclient.InferRequestedOutput("audio")], + request_id=request_id, + ) + + t0 = time.perf_counter() + + start_inputs = [ + _str_input("stream_id", stream_id), + _bool_input("stream_start", True), + _str_input("speaker", speaker), + _str_input("context_text", context_text), + ] + if chunks[0]: + start_inputs.append(_token_input(chunks[0])) + if num_requests == 1: + start_inputs.append(_bool_input("stream_end", True)) + _send(start_inputs, start_rid) + + for ci, chunk in enumerate(chunks[1:], start=1): + if token_delay: + time.sleep(token_delay) + inputs = [_str_input("stream_id", stream_id), _token_input(chunk)] + if ci == num_requests - 1: + inputs.append(_bool_input("stream_end", True)) + _send(inputs, f"{stream_id}-c{ci}") + + num_samples = 0 + t_first: float | None = None + t_prev: float | None = None + keepup: list[float] = [] + audio_chunks: list[np.ndarray] = [] + reached_eos = False + + # All audio and the authoritative final ride on the stream_start response; + # follow-up token requests complete with an empty final that Triton does not + # surface to the client, so we key off start_rid only (matches the demo). + while True: + try: + result, error = result_q.get(timeout=chunk_timeout) + except queue.Empty: + elapsed = time.perf_counter() - t0 + return RequestResult(uttid=uttid, elapsed_s=elapsed, ttfa_s=elapsed, error="no chunk within chunk_timeout") + + if error: + elapsed = time.perf_counter() - t0 + return RequestResult(uttid=uttid, elapsed_s=elapsed, ttfa_s=elapsed, error=str(error)) + + response = result.get_response() + if response.id != start_rid: + continue + + audio = result.as_numpy("audio") + if audio is not None: + audio = audio.squeeze() + if audio.size > 0: + now = time.perf_counter() + if t_first is None: + t_first = now + else: + keepup.append((now - t_prev) / (audio.size / SAMPLE_RATE)) + t_prev = now + num_samples += int(audio.size) + if save_audio: + audio_chunks.append(np.asarray(audio, dtype=np.float32).reshape(-1)) + + if bool(getattr(response.parameters.get("triton_final_response"), "bool_param", False)): + reached_eos = True + break + + # Drain any straggler follow-up responses so they don't bleed into the next + # utterance on this persistent stream (cross-request leftovers are also + # filtered out by the start_rid check above, since each request uses a fresh id). + while True: + try: + result_q.get_nowait() + except queue.Empty: + break + + elapsed = time.perf_counter() - t0 + ttfa = (t_first - t0) if t_first is not None else elapsed + return RequestResult( + uttid=uttid, + num_samples=num_samples, + elapsed_s=elapsed, + ttfa_s=ttfa, + reached_eos=reached_eos, + keepup_ratios=keepup, + audio=(np.concatenate(audio_chunks) if save_audio and audio_chunks else None), + ) + + +def worker( + worker_id: int, + triton_url: str, + tokenizer, + items: list[tuple[str, str]], + task_queue: list[int], + queue_lock: threading.Lock, + stats: BenchmarkStats, + speaker: str, + context_text: str, + tokens_per_chunk: int, + token_delay: float, + chunk_timeout: float, + output_dir: Path | None, + verbose: bool, +): + result_q: queue.Queue = queue.Queue() + client = grpcclient.InferenceServerClient(url=triton_url) + client.start_stream(callback=lambda result, error: result_q.put((result, error))) + + try: + while True: + with queue_lock: + if not task_queue: + return + task_idx = task_queue.pop() + + uttid, text = random.choice(items) + result = synthesize_streaming( + client, + result_q, + tokenizer, + uttid, + text, + speaker, + context_text, + tokens_per_chunk, + token_delay, + chunk_timeout, + save_audio=output_dir is not None, + ) + stats.add(result) + + if result.error is not None: + client.stop_stream() + client.start_stream(callback=lambda result, error: result_q.put((result, error))) + if verbose: + print(f"[worker {worker_id:02d}] req {task_idx} ({uttid}) FAILED — {result.error}") + else: + if output_dir is not None and result.audio is not None and result.audio.size > 0: + _save_wav(output_dir / f"{uttid}.wav", result.audio) + if verbose: + print( + f"[worker {worker_id:02d}] req {task_idx} ({uttid}) — " + f"{result.num_samples / SAMPLE_RATE:.2f}s audio in {result.elapsed_s:.2f}s " + f"(TTFA {result.ttfa_s * 1000:.0f}ms)" + ) + finally: + client.stop_stream() + + +def _load_items(text_file: str) -> list[tuple[str, str]]: + items: list[tuple[str, str]] = [] + with open(text_file) as f: + for line in f: + line = line.rstrip("\n") + if not line.strip(): + continue + parts = line.split("\t", 1) + if len(parts) != 2: + raise ValueError(f"Expected '\\t' per line, got: {line!r}") + uttid, text = parts[0].strip(), parts[1].strip() + if not uttid or not text: + raise ValueError(f"Empty uttid or text in line: {line!r}") + items.append((uttid, text)) + return items + + +def _run_workers( + num_workers: int, + triton_url: str, + tokenizer, + items: list[tuple[str, str]], + num_tasks: int, + speaker: str, + context_text: str, + tokens_per_chunk: int, + token_delay: float, + chunk_timeout: float, + output_dir: Path | None, + verbose: bool, +) -> tuple[BenchmarkStats, float]: + task_queue = list(range(num_tasks)) + queue_lock = threading.Lock() + stats = BenchmarkStats() + + threads = [ + threading.Thread( + target=worker, + args=( + i, + triton_url, + tokenizer, + items, + task_queue, + queue_lock, + stats, + speaker, + context_text, + tokens_per_chunk, + token_delay, + chunk_timeout, + output_dir, + verbose, + ), + ) + for i in range(num_workers) + ] + wall_start = time.perf_counter() + for t in threads: + t.start() + for t in threads: + t.join() + return stats, time.perf_counter() - wall_start + + +def _percentile(sorted_vals: list[float], pct: float) -> float: + if not sorted_vals: + return 0.0 + idx = min(len(sorted_vals) - 1, int(len(sorted_vals) * pct)) + return sorted_vals[idx] + + +def _summarize(stats: BenchmarkStats, wall_s: float, concurrency: int) -> dict: + eos = [r for r in stats.results if r.reached_eos] + audio_s = sum(r.num_samples for r in eos) / SAMPLE_RATE + ttfas_ms = sorted(r.ttfa_s * 1000 for r in eos) + keepup = sorted(ratio for r in eos for ratio in r.keepup_ratios) + return { + "concurrency": concurrency, + "failed": len(stats.results) - len(eos), + "wall_s": wall_s, + "audio_s": audio_s, + "rtx": audio_s / wall_s if wall_s > 0 else 0.0, + "tput": len(eos) / wall_s if wall_s > 0 else 0.0, + "ttfa_mean_ms": (sum(ttfas_ms) / len(ttfas_ms)) if ttfas_ms else 0.0, + "ttfa_p95_ms": _percentile(ttfas_ms, 0.95), + "keepup_mean": (sum(keepup) / len(keepup)) if keepup else 0.0, + "keepup_p95": _percentile(keepup, 0.95), + } + + +def _print_summary(s: dict): + print( + f"[concurrency={s['concurrency']}] rtx = synt / wall = " + f"{s['rtx']:.2f}x = {s['audio_s']:.2f} / {s['wall_s']:.0f}" + ) + print( + f"throughput={s['tput']:.2f} req/s; failed = {s['failed']}; " + f"TTFA={s['ttfa_mean_ms']:.1f} / {s['ttfa_p95_ms']:.1f} (p95); " + f"keepup={s['keepup_mean']:.3f} / {s['keepup_p95']:.3f} (p95)" + ) + + +def main(): + parser = argparse.ArgumentParser(description="Benchmark EasyMagpie TTS Triton server (streaming-text mode)") + parser.add_argument("--text-file", required=True, help="Path to file with '\\t' per line") + parser.add_argument("--model-dir", required=True, help="Model dir with the tokenizer (text -> subword ids)") + parser.add_argument("-n", "--num-requests", type=int, required=True, help="Requests per concurrency level") + parser.add_argument("-c", "--concurrency", type=int, nargs="+", default=[4], help="Concurrency levels to test") + parser.add_argument("--triton-url", default="localhost:8001", help="Triton gRPC endpoint (default localhost:8001)") + parser.add_argument("--speaker", default="eng", help="Speaker id (default: eng)") + parser.add_argument("--context-text", default="[EN]", help="Context text (default: [EN])") + parser.add_argument("--tokens-per-chunk", type=int, default=1, help="Subword ids fed per stream chunk (default 1)") + parser.add_argument( + "--token-delay", type=float, default=0.0, help="Sleep between token chunks to mimic upstream LLM (default 0)" + ) + parser.add_argument("--no-warmup", action="store_true", help="Skip warmup phase (3 requests per worker)") + parser.add_argument("--chunk-timeout", type=float, default=60, help="Per-chunk receive timeout, s (default: 60)") + parser.add_argument( + "--output-dir", default=None, help="If set, write each generated waveform to /.wav" + ) + parser.add_argument("--verbose", action="store_true", help="Print per-request lines") + args = parser.parse_args() + + output_dir = Path(args.output_dir) if args.output_dir else None + if output_dir is not None: + output_dir.mkdir(parents=True, exist_ok=True) + + items = _load_items(args.text_file) + if not items: + print(f"ERROR: no usable lines found in {args.text_file}") + return + + print(f"Loading tokenizer from {args.model_dir} ...") + tokenizer = AutoTokenizer.from_pretrained(args.model_dir, trust_remote_code=True) + print( + f"Loaded {len(items)} utterances; {args.num_requests} req/level; concurrency {args.concurrency} " + f"({args.tokens_per_chunk} tok/chunk, {args.token_delay}s delay)" + ) + + summaries = [] + for concurrency in args.concurrency: + if not args.no_warmup: + _run_workers( + concurrency, + args.triton_url, + tokenizer, + items, + concurrency * 3, + args.speaker, + args.context_text, + args.tokens_per_chunk, + args.token_delay, + args.chunk_timeout, + output_dir=None, + verbose=False, + ) + + stats, wall_elapsed = _run_workers( + concurrency, + args.triton_url, + tokenizer, + items, + args.num_requests, + args.speaker, + args.context_text, + args.tokens_per_chunk, + args.token_delay, + args.chunk_timeout, + output_dir=output_dir, + verbose=args.verbose, + ) + summary = _summarize(stats, wall_elapsed, concurrency) + summaries.append(summary) + _print_summary(summary) + + if len(summaries) > 1: + print("\n=== Summary ===") + for s in summaries: + _print_summary(s) + + +if __name__ == "__main__": + main() diff --git a/examples/tts/easymagpie_vllm_omni/model_repository/easymp/1/model.py b/examples/tts/easymagpie_vllm_omni/model_repository/easymp/1/model.py index a5f04ead6b7d..202cc488e4da 100644 --- a/examples/tts/easymagpie_vllm_omni/model_repository/easymp/1/model.py +++ b/examples/tts/easymagpie_vllm_omni/model_repository/easymp/1/model.py @@ -17,16 +17,24 @@ used by the inference demo / benchmark): it streams stacked codec frames, which we chunk-decode (overlap-save) through the ``codec`` TensorRT model. -Pipeline: - 1. Build ``additional_information`` from ``{speaker_embedding, context_text, text, - temperature, top_k}`` and a placeholder ``prompt_token_ids`` of length - ``estimate_prompt_len(...)``. - 2. Submit one request to ``AsyncOmni.generate()``. Each step yields the - *cumulative* ``audio_codes`` tensor ``(T_total, C*S)`` (prefill rows + one row - per decode step) and cumulative backbone ``token_ids``; we slice the decoded - rows, drop the leading ``speech_delay`` warm-up frames, and stop at the audio - EOS frame. - 3. New frames are streamed out in fixed ``codec_chunk_size``-frame windows (with a +Two request flavours share one engine (``async_chunk=True`` + +``EasyMagpieARAsyncScheduler``, a drop-in for both paths): + +* **whole-text** (default) — a request carries the full ``text``; we build a single + prompt and run ``AsyncOmni.generate(prompt, ...)``. +* **streaming-text** — the client pushes subword ``text_token`` ids one (or a few) + at a time across several requests sharing a ``stream_id`` (``stream_start`` on the + first, ``stream_end`` on the last). We feed those tokens as ``StreamingInput`` + chunks (prefill, then one chunk per subword with ``max_tokens=1``, then a free- + running acoustic tail) into a single ``AsyncOmni.generate(, ...)`` call. + All audio for the stream is sent back on the ``stream_start`` request's response + sender; the follow-up requests just feed tokens and close with no output. + +Both flavours converge on the same accumulator/codec pipeline: + 1. Each engine step yields the *cumulative* ``audio_codes`` ``(T_total, C*S)`` + (prefill rows + one row per decode step); we slice the decoded rows, drop the + leading ``speech_delay`` warm-up frames, and stop at the audio EOS frame. + 2. New frames are streamed out in fixed ``codec_chunk_size``-frame windows (with a trimmed ``codec_left_context``) through the ``codec`` BLS, which unstacks + index-converts + decodes them to 22.05 kHz audio chunks. """ @@ -57,6 +65,37 @@ ) logger = logging.getLogger("easymp_triton") +# Sentinel pushed onto a streaming session's token queue to signal "no more text" +# (``stream_end``): the input generator then appends text-EOS + the mask sentinel. +_STREAM_END = object() + + +class _StreamingSession: + """Per-``stream_id`` state for a streaming-text request. + + ``response_sender`` is the ``stream_start`` request's sender; *all* audio for the + stream is sent there. ``token_q`` is fed (thread-safely, via the event loop) by + the follow-up chunk requests and drained by the input async-generator. + + ``pace_q`` is the output->input back-pressure channel: vLLM drains the input + async-generator eagerly (a background task, decoupled from decode steps), and the + scheduler *replaces* ``additional_information`` per chunk, so feeding faster than + the model decodes overwrites not-yet-consumed ``text_token``s. We therefore + release exactly one chunk per observed decode-step output: ``_drive_codec`` puts a + token here after each step and ``_stream_inputs`` waits for one before each yield. + """ + + __slots__ = ("stream_id", "request_id", "speaker", "context_text", "response_sender", "token_q", "pace_q") + + def __init__(self, stream_id, request_id, speaker, context_text, response_sender, token_q, pace_q): + self.stream_id = stream_id + self.request_id = request_id + self.speaker = speaker + self.context_text = context_text + self.response_sender = response_sender + self.token_q = token_q + self.pace_q = pace_q + def _require_param(parameters: dict, key: str) -> str: val = parameters.get(key) @@ -92,10 +131,17 @@ def initialize(self, args): self.lt_top_k = int(_require_param(params, "lt_top_k")) self._load_arch_and_tokenizer() + self._init_sampling_helpers() self._speaker_cache: dict = {} # Inferred from the first codec decode (audio_len / codec_chunk_size). self._spf: int | None = None + # Active streaming-text sessions keyed by stream_id. Touched from the Triton + # execute() thread (add/lookup) and the asyncio loop thread (removal on + # completion), so guard it with a lock. + self._sessions: dict[str, _StreamingSession] = {} + self._sessions_lock = threading.Lock() + self._loop = asyncio.new_event_loop() self._loop_thread = threading.Thread(target=self._loop.run_forever, daemon=True) self._loop_thread.start() @@ -126,12 +172,48 @@ def _load_arch_and_tokenizer(self): self.num_stacked_codebooks = int(arch.num_stacked_codebooks) self.has_task_embedding = arch.num_task_embeddings > 0 self.stop_token_id = EasyMagpieTTSForConditionalGeneration.audio_eos_stop_token_id(cfg_obj) + # Appended after the client's streamed subword ids to close the text channel + # before the free-running acoustic tail (matches the demo / benchmark). + self.text_eos_id = int(config.get("text_vocab_size", config.get("vocab_size", 0))) - 2 self.tokenizer = AutoTokenizer.from_pretrained(self.vllm_model_path, trust_remote_code=True) self._estimate_prompt_len = EasyMagpieTTSForConditionalGeneration.estimate_prompt_len + def _init_sampling_helpers(self): + """Cache the vLLM sampling/streaming types used by both request flavours.""" + from vllm import SamplingParams + from vllm.sampling_params import RequestOutputKind + + try: + from vllm.engine.protocol import StreamingInput + except ImportError: + from vllm.v1.engine.async_llm import StreamingInput + + self._SamplingParams = SamplingParams + self._RequestOutputKind = RequestOutputKind + self._StreamingInput = StreamingInput + + def _make_sampling_params(self, max_tokens: int): + """SamplingParams shared by both paths (audio sampling happens in the LT). + + ``output_kind=DELTA`` is what makes ``audio_codes`` arrive as a growing list + of per-step frames during decode; the backbone token sampler is a no-op + (temperature 0) and stops at the audio-EOS ``stop_token_id``. + """ + return self._SamplingParams( + temperature=0.0, + max_tokens=max(1, int(max_tokens)), + detokenize=False, + ignore_eos=True, + stop_token_ids=[self.stop_token_id], + output_kind=self._RequestOutputKind.DELTA, + ) + def _build_stage_config_file(self) -> str: stage_cfg = { + # async_chunk enables the streaming-text feed (one subword per chunk); + # it is a no-op for the whole-text path, so one engine serves both. + "async_chunk": True, "stage_args": [ { "stage_id": 0, @@ -145,7 +227,11 @@ def _build_stage_config_file(self) -> str: "max_num_seqs": self.max_num_seqs, "model_arch": "EasyMagpieTTSForConditionalGeneration", "worker_type": "ar", - "scheduler_cls": "vllm_omni.core.sched.omni_ar_scheduler.OmniARAsyncScheduler", + # EasyMagpie-aware scheduler: forwards each chunk's text_token + # and the raised acoustic-tail max_tokens for streaming-text, + # and is a drop-in equivalent of the stock scheduler for + # whole-text. + "scheduler_cls": "easymagpie_vllm_omni.scheduler.EasyMagpieARAsyncScheduler", "enforce_eager": False, "trust_remote_code": True, "async_scheduling": True, @@ -204,14 +290,19 @@ def _get_speaker_embedding(self, speaker: str) -> torch.Tensor: self._speaker_cache[speaker] = emb.to(torch.float32) return self._speaker_cache[speaker] + def _prompt_len(self, speaker_embedding: torch.Tensor, context_text: str) -> int: + return int( + self._estimate_prompt_len( + speaker_embedding, + tokenize=lambda t: self.tokenizer.encode(t), + context_text=context_text, + has_task_embedding=self.has_task_embedding, + ) + ) + def _build_prompt(self, text: str, context_text: str, speaker: str) -> dict: speaker_embedding = self._get_speaker_embedding(speaker) - prompt_len = self._estimate_prompt_len( - speaker_embedding, - tokenize=lambda t: self.tokenizer.encode(t), - context_text=context_text, - has_task_embedding=self.has_task_embedding, - ) + prompt_len = self._prompt_len(speaker_embedding, context_text) return { "prompt_token_ids": [0] * prompt_len, "additional_information": { @@ -291,28 +382,59 @@ def _codec_worker(self, codec_q: queue.Queue, response_sender, state: dict) -> N if not finalized: self._send_error(response_sender, e) - async def _synthesize(self, text: str, context_text: str, speaker: str, response_sender): + @staticmethod + def _cumulative_codes(payload): + """Reduce one step's ``audio_codes`` payload to the cumulative ``(T, C*S)``. + + DELTA decode surfaces a growing list ``[cum_so_far, new_frame, ...]``, but + every finished segment consolidates to a single cumulative tensor — and with + ``max_tokens=1`` (streaming-text) *each* fed token finishes its segment, so + most steps arrive as a tensor, not a list. Both forms reduce to the full + cumulative here (cat the list / take the tensor); callers keep the largest. + """ + if isinstance(payload, list): + parts = [t for t in payload if isinstance(t, torch.Tensor) and t.numel() > 0] + return torch.cat(parts, dim=0) if parts else None + if isinstance(payload, torch.Tensor) and payload.numel() > 0: + return payload + return None + + async def _drive_codec( + self, gen, response_sender, request_id: str, speaker: str, text: str, prompt_len: int, pace_q=None + ): + """Drain one omni request's per-step ``audio_codes`` and stream audio out. + + ``gen`` is the ``AsyncOmni.generate(...)`` async iterator (whole-text or + streaming-text); from here on both flavours are identical. All audio is sent + on ``response_sender`` (for streaming that is the ``stream_start`` sender). + + Each step yields the *cumulative* codes ``(prompt_len prefill rows + decoded + frames, C*S)`` (as a list to cat or an already-consolidated tensor); the first + real audio frame is at row ``prompt_len + speech_delay`` and the last decoded + row is the audio-EOS frame. + + For streaming-text, ``pace_q`` carries one token per observed decode-step + output so the input feeder releases the next chunk only after the previous one + has been decoded (see :class:`_StreamingSession`); ``None`` for whole-text. + """ t_start = time.perf_counter() - request_id = f"easymp-{uuid.uuid4().hex[:8]}" codec_q: queue.Queue = queue.Queue() state: dict = {"t_first_audio": None, "error": None} codec_future = self._codec_pool.submit(self._codec_worker, codec_q, response_sender, state) - # The omni accumulator yields a tensor for the prefill (first) and the - # consolidated (final) steps, but a growing list during decode (one - # (1, C*S) row appended per AR step). We only consume the list yields; - # ``mm_codes[0]`` is the prefill prefix, ``mm_codes[1 + d]`` is decode - # frame d. Real audio starts after ``speech_delay`` warm-up frames. + # Codes are a cumulative ``(T, C*S)`` tensor: rows [0, prompt_len) are the + # prefill prefix, the next ``speech_delay`` rows are warm-up, so the first + # real audio frame is at ``head`` and the last decoded row is audio-EOS. L = self.codec_left_context - base = 1 + self.speech_delay # mm_codes index of the first real frame + head = prompt_len + self.speech_delay # cumulative row of the first real frame sent = 0 # real frames already queued to the codec threshold = self.first_chunk_frames - mm_codes: list | None = None + cum = None # largest cumulative codes tensor seen produced_final = False - def emit_ready(codes_list: list, real_count: int, final: bool) -> None: - """Queue overlap-save windows of newly-ready real frames.""" + def emit_ready(cum_codes, real_count: int, final: bool) -> None: + """Queue overlap-save windows of newly-ready real frames (by cum row).""" nonlocal sent, threshold, produced_final while sent < real_count: remaining = real_count - sent @@ -320,39 +442,58 @@ def emit_ready(codes_list: list, real_count: int, final: bool) -> None: break take = min(threshold, remaining) ctx = min(sent, L) - chunk = torch.cat(codes_list[base + sent - ctx : base + sent + take], dim=0) + chunk = cum_codes[head + sent - ctx : head + sent + take] sent += take threshold = self.codec_chunk_size - L is_final = final and sent >= real_count codec_q.put((chunk, ctx, is_final)) produced_final = produced_final or is_final + step = 0 try: - prompt = self._build_prompt(text, context_text, speaker) - async for out in self.omni.generate(prompt, request_id=request_id): + async for out in gen: + # Release the next input chunk for each decode-step output (one chunk + # produces one frame); harmless extra puts during the acoustic tail. + if pace_q is not None: + pace_q.put_nowait(True) if state["error"] is not None: break + step += 1 payload = (getattr(out, "multimodal_output", None) or {}).get("audio_codes") - if not isinstance(payload, list): + cum_now = self._cumulative_codes(payload) + if cum_now is None: continue - mm_codes = payload - # Hold back the most recent decode frame: the audio-EOS frame is - # always the last one, and must not be vocoded. - real_avail = (len(mm_codes) - 1) - self.speech_delay - 1 + if cum is None or cum_now.shape[0] > cum.shape[0]: + cum = cum_now + # Hold back the most recent decode row: the audio-EOS frame is always + # the last one and must not be vocoded. + real_avail = cum.shape[0] - head - 1 + if pace_q is not None: + logger.info( + "rid=%s STEP %d: got acoustic codes (cum_rows=%d, real_avail=%d, sent=%d) -> released pace", + request_id, + step, + cum.shape[0], + max(0, real_avail), + sent, + ) if real_avail > sent: - emit_ready(mm_codes, real_avail, final=False) + before = sent + emit_ready(cum, real_avail, final=False) + if pace_q is not None and sent > before: + logger.info("rid=%s STEP %d: queued %d new frame(s) to codec", request_id, step, sent - before) - if state["error"] is None and mm_codes is not None: - # Authoritative tail: scan for the audio-EOS frame (only it carries + if state["error"] is None and cum is not None: + # Authoritative tail: scan for the audio-EOS row (only it carries # audio_eos_id > codebook_size) and vocode every real frame before it. - eos_idx = None - for i in range(len(mm_codes) - 1, 0, -1): - if bool((mm_codes[i] == self.audio_eos_id).any()): - eos_idx = i + eos_row = None + for i in range(cum.shape[0] - 1, head - 1, -1): + if bool((cum[i] == self.audio_eos_id).any()): + eos_row = i break - last_excl = eos_idx if eos_idx is not None else len(mm_codes) - real_count = (last_excl - 1) - self.speech_delay - emit_ready(mm_codes, real_count, final=True) + last_excl = eos_row if eos_row is not None else cum.shape[0] + real_count = max(0, last_excl - head) + emit_ready(cum, real_count, final=True) if not produced_final: codec_q.put(None) elif state["error"] is None: @@ -386,6 +527,121 @@ def emit_ready(codes_list: list, real_count: int, final: bool) -> None: except Exception: pass self._send_error(response_sender, e) + finally: + try: + await gen.aclose() + except Exception: + pass + + async def _synthesize_whole_text(self, text: str, context_text: str, speaker: str, response_sender): + request_id = f"easymp-{uuid.uuid4().hex[:8]}" + prompt = self._build_prompt(text, context_text, speaker) + prompt_len = len(prompt["prompt_token_ids"]) + gen = self.omni.generate( + prompt, + sampling_params_list=[self._make_sampling_params(self.max_new_tokens)], + request_id=request_id, + ) + await self._drive_codec(gen, response_sender, request_id, speaker, text, prompt_len) + + def _text_chunk(self, text_token: int, sampling_params): + return self._StreamingInput( + prompt={"prompt_token_ids": [0], "additional_information": {"text_token": int(text_token)}}, + sampling_params=sampling_params, + ) + + async def _stream_inputs(self, session: _StreamingSession): + """Yield ``StreamingInput`` chunks for a streaming-text session. + + Prefill (speaker + context, no text), then one ``max_tokens=1`` chunk per + subword id, then — once the client signals ``stream_end`` — the text-EOS id + and a ``-1`` mask sentinel whose raised ``max_tokens`` lets the model free-run + the acoustic tail to audio-EOS. + + Every post-prefill chunk is gated on ``session.pace_q`` (one decode-step + output == one chunk released) so the eagerly-drained feed can't overwrite a + not-yet-consumed ``text_token``; content tokens are additionally gated on the + client actually having sent them. + """ + StreamingInput = self._StreamingInput + rid = session.request_id + speaker_embedding = self._get_speaker_embedding(session.speaker) + prompt_len = self._prompt_len(speaker_embedding, session.context_text) + sp1 = self._make_sampling_params(1) + + prefill_info = { + "speaker_embedding": speaker_embedding, + "context_text": session.context_text, + "temperature": self.lt_temperature, + "top_k": self.lt_top_k, + } + # Prefill is released immediately; its decode-step output unblocks the first + # text token (mirrors the demo's go_queue handshake). + logger.info("rid=%s STREAM: releasing prefill (prompt_len=%d, speaker=%s)", rid, prompt_len, session.speaker) + yield StreamingInput( + prompt={"prompt_token_ids": [0] * prompt_len, "additional_information": prefill_info}, + sampling_params=sp1, + ) + + n_text = 0 + pending: list = [] + ended = False + while True: + # Refill from the client; block only while we have nothing buffered. + while not pending and not ended: + logger.info("rid=%s STREAM: waiting for client text tokens...", rid) + item = await session.token_q.get() + if item is _STREAM_END: + ended = True + logger.info("rid=%s STREAM: client signalled stream_end", rid) + else: + pending.extend(int(t) for t in item) + logger.info("rid=%s STREAM: client sent tokens=%s (buffered=%d)", rid, list(item), len(pending)) + if not pending: + break + await session.pace_q.get() + tok = pending.pop(0) + logger.info( + "rid=%s STREAM: feeding text_token=%d (n_text=%d, buffered_left=%d)", rid, tok, n_text, len(pending) + ) + yield self._text_chunk(tok, sp1) + n_text += 1 + + await session.pace_q.get() + logger.info("rid=%s STREAM: feeding text_eos=%d (n_text=%d)", rid, self.text_eos_id, n_text) + yield self._text_chunk(self.text_eos_id, sp1) + n_text += 1 + + await session.pace_q.get() + tail_budget = self.max_new_tokens - n_text + logger.info("rid=%s STREAM: feeding mask sentinel text_token=-1 (acoustic tail budget=%d)", rid, tail_budget) + tail_params = self._make_sampling_params(tail_budget) + yield self._text_chunk(-1, tail_params) + logger.info("rid=%s STREAM: input generator exhausted (fed %d text tokens incl. eos)", rid, n_text) + + async def _synthesize_streaming(self, session: _StreamingSession): + prompt_len = self._prompt_len(self._get_speaker_embedding(session.speaker), session.context_text) + inputs_gen = self._stream_inputs(session) + gen = self.omni.generate( + inputs_gen, sampling_params_list=[self._make_sampling_params(1)], request_id=session.request_id + ) + try: + await self._drive_codec( + gen, + session.response_sender, + session.request_id, + session.speaker, + "", + prompt_len, + pace_q=session.pace_q, + ) + finally: + try: + await inputs_gen.aclose() + except Exception: + pass + with self._sessions_lock: + self._sessions.pop(session.stream_id, None) @staticmethod def _log_future_exception(future: concurrent.futures.Future) -> None: @@ -394,7 +650,7 @@ def _log_future_exception(future: concurrent.futures.Future) -> None: except concurrent.futures.CancelledError: return if exc is not None: - logger.error("_synthesize task crashed: %s", exc, exc_info=exc) + logger.error("synthesis task crashed: %s", exc, exc_info=exc) @staticmethod def _read_str(request, name: str, default: str) -> str: @@ -403,23 +659,100 @@ def _read_str(request, name: str, default: str) -> str: return default return tensor.as_numpy().flatten()[0].decode("utf-8") + @staticmethod + def _read_bool(request, name: str, default: bool) -> bool: + tensor = pb_utils.get_input_tensor_by_name(request, name) + if tensor is None: + return default + return bool(tensor.as_numpy().flatten()[0]) + + @staticmethod + def _read_int_list(request, name: str) -> list: + tensor = pb_utils.get_input_tensor_by_name(request, name) + if tensor is None: + return [] + return [int(x) for x in tensor.as_numpy().flatten().tolist()] + + def _feed_session(self, session: _StreamingSession, item) -> None: + """Hand ``item`` (a list of token ids or ``_STREAM_END``) to the session's + queue on the event-loop thread (asyncio.Queue is not thread-safe).""" + self._loop.call_soon_threadsafe(session.token_q.put_nowait, item) + + def _launch(self, coro) -> None: + future = asyncio.run_coroutine_threadsafe(coro, self._loop) + future.add_done_callback(self._log_future_exception) + def execute(self, requests): for request in requests: response_sender = request.get_response_sender() try: - text = self._read_str(request, "text", "") - context_text = self._read_str(request, "context_text", self.default_context_text) - speaker = self._read_str(request, "speaker", self.default_speaker) - future = asyncio.run_coroutine_threadsafe( - self._synthesize(text, context_text, speaker, response_sender), - self._loop, - ) - future.add_done_callback(self._log_future_exception) + stream_id = self._read_str(request, "stream_id", "") + if stream_id: + self._handle_streaming(request, stream_id, response_sender) + else: + self._handle_whole_text(request, response_sender) except Exception as e: logger.error("Request parse failed: %s", e, exc_info=True) self._send_error(response_sender, e) return None + def _handle_whole_text(self, request, response_sender) -> None: + text = self._read_str(request, "text", "") + context_text = self._read_str(request, "context_text", self.default_context_text) + speaker = self._read_str(request, "speaker", self.default_speaker) + self._launch(self._synthesize_whole_text(text, context_text, speaker, response_sender)) + + def _handle_streaming(self, request, stream_id: str, response_sender) -> None: + """Route one chunk of a streaming-text request. + + ``stream_start`` opens a session (launching one generate() call whose audio + streams back on *this* response sender) and ``stream_end`` closes the text + feed. Follow-up chunks only push tokens, then immediately complete their own + (output-less) response so the client side of the stream stays tidy. + """ + start = self._read_bool(request, "stream_start", False) + end = self._read_bool(request, "stream_end", False) + tokens = self._read_int_list(request, "text_token") + logger.info( + "CLIENT chunk: stream_id=%s start=%s end=%s tokens=%s", stream_id, start, end, tokens + ) + + if start: + context_text = self._read_str(request, "context_text", self.default_context_text) + speaker = self._read_str(request, "speaker", self.default_speaker) + session = _StreamingSession( + stream_id=stream_id, + request_id=f"easymp-stream-{uuid.uuid4().hex[:8]}", + speaker=speaker, + context_text=context_text, + response_sender=response_sender, + token_q=asyncio.Queue(), + pace_q=asyncio.Queue(), + ) + with self._sessions_lock: + self._sessions[stream_id] = session + logger.info("rid=%s STREAM: opened session for stream_id=%s", session.request_id, stream_id) + # All audio for the stream is sent on this (stream_start) sender by the + # coroutine; do NOT complete it here. + self._launch(self._synthesize_streaming(session)) + if tokens: + self._feed_session(session, tokens) + if end: + self._feed_session(session, _STREAM_END) + return + + with self._sessions_lock: + session = self._sessions.get(stream_id) + if session is not None: + if tokens: + self._feed_session(session, tokens) + if end: + self._feed_session(session, _STREAM_END) + else: + logger.warning("Streaming chunk for unknown/closed stream_id=%s", stream_id) + # Follow-up chunks produce no audio of their own; close this response. + response_sender.send(flags=pb_utils.TRITONSERVER_RESPONSE_COMPLETE_FINAL) + def finalize(self): if hasattr(self, "omni"): try: diff --git a/examples/tts/easymagpie_vllm_omni/model_repository/easymp/config.pbtxt b/examples/tts/easymagpie_vllm_omni/model_repository/easymp/config.pbtxt index bc5f70d3bcf0..b5d58f6a0707 100644 --- a/examples/tts/easymagpie_vllm_omni/model_repository/easymp/config.pbtxt +++ b/examples/tts/easymagpie_vllm_omni/model_repository/easymp/config.pbtxt @@ -2,11 +2,20 @@ name: "easymp" backend: "python" max_batch_size: 32 +# Two request flavours share this model: +# * whole-text : send "text" (+ optional context_text/speaker). +# * streaming-text: send "stream_id" plus "text_token" chunks across several +# requests; mark the first with stream_start=true and the last +# with stream_end=true. context_text/speaker are read from the +# stream_start request. All audio is streamed back on the +# stream_start request's response; follow-up chunk requests get +# an empty final response. input [ { name: "text" data_type: TYPE_STRING dims: [ 1 ] + optional: true }, { name: "context_text" @@ -19,6 +28,30 @@ input [ data_type: TYPE_STRING dims: [ 1 ] optional: true + }, + { + name: "stream_id" + data_type: TYPE_STRING + dims: [ 1 ] + optional: true + }, + { + name: "text_token" + data_type: TYPE_INT32 + dims: [ -1 ] + optional: true + }, + { + name: "stream_start" + data_type: TYPE_BOOL + dims: [ 1 ] + optional: true + }, + { + name: "stream_end" + data_type: TYPE_BOOL + dims: [ 1 ] + optional: true } ] diff --git a/examples/tts/easymagpie_vllm_omni/run_service_request.ipynb b/examples/tts/easymagpie_vllm_omni/run_service_request.ipynb index 3517294521fe..0e5888dee573 100644 --- a/examples/tts/easymagpie_vllm_omni/run_service_request.ipynb +++ b/examples/tts/easymagpie_vllm_omni/run_service_request.ipynb @@ -107,11 +107,160 @@ "\n", "display(Audio(audio, rate=SAMPLE_RATE))" ] + }, + { + "cell_type": "markdown", + "id": "b4a7d7cf", + "metadata": {}, + "source": [ + "## Streaming-text input\n", + "\n", + "Instead of one request with the full `text`, push subword ids a few at a time over\n", + "a single gRPC stream. Every chunk shares a `stream_id`; mark the first request with\n", + "`stream_start=true` (it also carries `speaker` / `context_text`) and the last with\n", + "`stream_end=true`. The server appends text-EOS + a free-running acoustic tail, and\n", + "streams **all** audio back on the `stream_start` request's response (follow-up chunk\n", + "requests just feed tokens and return an empty final response). This is the\n", + "live-client path: synthesis begins before the whole text is known." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "79da2141", + "metadata": {}, + "outputs": [], + "source": [ + "import uuid\n", + "\n", + "from transformers import AutoTokenizer\n", + "\n", + "# Same tokenizer the converted model bundles; used to turn text into the subword\n", + "# ids the streaming path expects (specials off — the server appends text-EOS).\n", + "MODEL_DIR = \"/home/vklimkov/workspace/emp/NeMo/examples/tts/easymagpie_vllm_omni/easymp_vllm_model\"\n", + "tokenizer = AutoTokenizer.from_pretrained(MODEL_DIR, trust_remote_code=True)\n", + "\n", + "\n", + "def _str_input(name: str, value: str):\n", + " t = grpcclient.InferInput(name, [1, 1], \"BYTES\")\n", + " t.set_data_from_numpy(np.array([[value]], dtype=object))\n", + " return t\n", + "\n", + "\n", + "def _bool_input(name: str, value: bool):\n", + " t = grpcclient.InferInput(name, [1, 1], \"BOOL\")\n", + " t.set_data_from_numpy(np.array([[value]], dtype=bool))\n", + " return t\n", + "\n", + "\n", + "def _token_input(tokens):\n", + " t = grpcclient.InferInput(\"text_token\", [1, len(tokens)], \"INT32\")\n", + " t.set_data_from_numpy(np.array([tokens], dtype=np.int32))\n", + " return t\n", + "\n", + "\n", + "def synthesize_streaming(\n", + " text: str,\n", + " speaker: str = \"eng\",\n", + " context_text: str = \"[EN]\",\n", + " tokens_per_chunk: int = 1,\n", + " delay_s: float = 0.0,\n", + ") -> np.ndarray:\n", + " \"\"\"Stream subword ids ``tokens_per_chunk`` at a time; collect streamed audio.\n", + "\n", + " ``delay_s`` (optional) sleeps between chunks to mimic tokens trickling in from\n", + " an upstream producer. Audio arrives on the ``stream_start`` request's response.\n", + " \"\"\"\n", + " token_ids = tokenizer.encode(text, add_special_tokens=False)\n", + " chunks = [token_ids[i : i + tokens_per_chunk] for i in range(0, len(token_ids), tokens_per_chunk)] or [[]]\n", + "\n", + " stream_id = uuid.uuid4().hex\n", + " start_rid = f\"{stream_id}-start\"\n", + " result_q: queue.Queue = queue.Queue()\n", + "\n", + " client = grpcclient.InferenceServerClient(url=TRITON_URL)\n", + " client.start_stream(callback=lambda result, error: result_q.put((result, error)))\n", + "\n", + " def _send(inputs, request_id):\n", + " client.async_stream_infer(\n", + " model_name=MODEL_NAME,\n", + " inputs=inputs,\n", + " outputs=[grpcclient.InferRequestedOutput(\"audio\")],\n", + " request_id=request_id,\n", + " )\n", + "\n", + " # stream_start: speaker/context (+ first token chunk). stream_end rides along if\n", + " # the whole utterance is a single chunk.\n", + " start_inputs = [\n", + " _str_input(\"stream_id\", stream_id),\n", + " _bool_input(\"stream_start\", True),\n", + " _str_input(\"speaker\", speaker),\n", + " _str_input(\"context_text\", context_text),\n", + " ]\n", + " if chunks[0]:\n", + " start_inputs.append(_token_input(chunks[0]))\n", + " if len(chunks) == 1:\n", + " start_inputs.append(_bool_input(\"stream_end\", True))\n", + " _send(start_inputs, start_rid)\n", + "\n", + " for ci, chunk in enumerate(chunks[1:], start=1):\n", + " if delay_s:\n", + " time.sleep(delay_s)\n", + " inputs = [_str_input(\"stream_id\", stream_id), _token_input(chunk)]\n", + " if ci == len(chunks) - 1:\n", + " inputs.append(_bool_input(\"stream_end\", True))\n", + " _send(inputs, f\"{stream_id}-c{ci}\")\n", + "\n", + " audio_chunks = []\n", + " t0 = time.perf_counter()\n", + " t_first = None\n", + " while True:\n", + " result, error = result_q.get(timeout=120)\n", + " if error:\n", + " client.stop_stream()\n", + " raise RuntimeError(str(error))\n", + " response = result.get_response()\n", + " # Audio only ever rides on the stream_start request's response.\n", + " if response.id != start_rid:\n", + " continue\n", + " audio = result.as_numpy(\"audio\")\n", + " if audio is not None:\n", + " audio = audio.squeeze()\n", + " if audio.size > 0:\n", + " if t_first is None:\n", + " t_first = time.perf_counter()\n", + " audio_chunks.append(audio)\n", + " final_param = response.parameters.get(\"triton_final_response\")\n", + " if final_param and getattr(final_param, \"bool_param\", False):\n", + " break\n", + "\n", + " client.stop_stream()\n", + " elapsed = time.perf_counter() - t0\n", + " ttfa = (t_first - t0) if t_first else elapsed\n", + " print(f\"Streamed {len(chunks)} text chunks | {len(audio_chunks)} audio chunks | TTFA: {ttfa:.3f}s | total: {elapsed:.3f}s\")\n", + " return np.concatenate(audio_chunks) if audio_chunks else np.array([], dtype=np.float32)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "97cd6444", + "metadata": {}, + "outputs": [], + "source": [ + "audio_stream = synthesize_streaming(\n", + " text=\"Since then physicists have found that it is not reflection, but refraction by the raindrops which causes the unicorns to fly.\",\n", + " tokens_per_chunk=1,\n", + " delay_s=0.003, # set e.g. 0.02 to simulate tokens trickling in from an upstream LLM\n", + ")\n", + "print(f\"Got {len(audio_stream)} samples — {len(audio_stream) / SAMPLE_RATE:.2f}s @ {SAMPLE_RATE} Hz\")\n", + "display(Audio(audio_stream, rate=SAMPLE_RATE))" + ] } ], "metadata": { "kernelspec": { - "display_name": "streaming_vllm", + "display_name": "emp", "language": "python", "name": "python3" }, @@ -125,7 +274,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.12" + "version": "3.12.13" } }, "nbformat": 4, From 9cf98c9e93939e6a9fac2c6643de75cb752559ed Mon Sep 17 00:00:00 2001 From: Viacheslav Klimkov Date: Wed, 10 Jun 2026 14:18:07 +0200 Subject: [PATCH 33/45] examples/tts/easymagpie_vllm_omni: clean up service logs Signed-off-by: Viacheslav Klimkov --- .../model_repository/easymp/1/model.py | 29 ------------------- 1 file changed, 29 deletions(-) diff --git a/examples/tts/easymagpie_vllm_omni/model_repository/easymp/1/model.py b/examples/tts/easymagpie_vllm_omni/model_repository/easymp/1/model.py index 202cc488e4da..1d53b8375225 100644 --- a/examples/tts/easymagpie_vllm_omni/model_repository/easymp/1/model.py +++ b/examples/tts/easymagpie_vllm_omni/model_repository/easymp/1/model.py @@ -449,7 +449,6 @@ def emit_ready(cum_codes, real_count: int, final: bool) -> None: codec_q.put((chunk, ctx, is_final)) produced_final = produced_final or is_final - step = 0 try: async for out in gen: # Release the next input chunk for each decode-step output (one chunk @@ -458,7 +457,6 @@ def emit_ready(cum_codes, real_count: int, final: bool) -> None: pace_q.put_nowait(True) if state["error"] is not None: break - step += 1 payload = (getattr(out, "multimodal_output", None) or {}).get("audio_codes") cum_now = self._cumulative_codes(payload) if cum_now is None: @@ -468,20 +466,8 @@ def emit_ready(cum_codes, real_count: int, final: bool) -> None: # Hold back the most recent decode row: the audio-EOS frame is always # the last one and must not be vocoded. real_avail = cum.shape[0] - head - 1 - if pace_q is not None: - logger.info( - "rid=%s STEP %d: got acoustic codes (cum_rows=%d, real_avail=%d, sent=%d) -> released pace", - request_id, - step, - cum.shape[0], - max(0, real_avail), - sent, - ) if real_avail > sent: - before = sent emit_ready(cum, real_avail, final=False) - if pace_q is not None and sent > before: - logger.info("rid=%s STEP %d: queued %d new frame(s) to codec", request_id, step, sent - before) if state["error"] is None and cum is not None: # Authoritative tail: scan for the audio-EOS row (only it carries @@ -564,7 +550,6 @@ async def _stream_inputs(self, session: _StreamingSession): client actually having sent them. """ StreamingInput = self._StreamingInput - rid = session.request_id speaker_embedding = self._get_speaker_embedding(session.speaker) prompt_len = self._prompt_len(speaker_embedding, session.context_text) sp1 = self._make_sampling_params(1) @@ -577,7 +562,6 @@ async def _stream_inputs(self, session: _StreamingSession): } # Prefill is released immediately; its decode-step output unblocks the first # text token (mirrors the demo's go_queue handshake). - logger.info("rid=%s STREAM: releasing prefill (prompt_len=%d, speaker=%s)", rid, prompt_len, session.speaker) yield StreamingInput( prompt={"prompt_token_ids": [0] * prompt_len, "additional_information": prefill_info}, sampling_params=sp1, @@ -589,35 +573,26 @@ async def _stream_inputs(self, session: _StreamingSession): while True: # Refill from the client; block only while we have nothing buffered. while not pending and not ended: - logger.info("rid=%s STREAM: waiting for client text tokens...", rid) item = await session.token_q.get() if item is _STREAM_END: ended = True - logger.info("rid=%s STREAM: client signalled stream_end", rid) else: pending.extend(int(t) for t in item) - logger.info("rid=%s STREAM: client sent tokens=%s (buffered=%d)", rid, list(item), len(pending)) if not pending: break await session.pace_q.get() tok = pending.pop(0) - logger.info( - "rid=%s STREAM: feeding text_token=%d (n_text=%d, buffered_left=%d)", rid, tok, n_text, len(pending) - ) yield self._text_chunk(tok, sp1) n_text += 1 await session.pace_q.get() - logger.info("rid=%s STREAM: feeding text_eos=%d (n_text=%d)", rid, self.text_eos_id, n_text) yield self._text_chunk(self.text_eos_id, sp1) n_text += 1 await session.pace_q.get() tail_budget = self.max_new_tokens - n_text - logger.info("rid=%s STREAM: feeding mask sentinel text_token=-1 (acoustic tail budget=%d)", rid, tail_budget) tail_params = self._make_sampling_params(tail_budget) yield self._text_chunk(-1, tail_params) - logger.info("rid=%s STREAM: input generator exhausted (fed %d text tokens incl. eos)", rid, n_text) async def _synthesize_streaming(self, session: _StreamingSession): prompt_len = self._prompt_len(self._get_speaker_embedding(session.speaker), session.context_text) @@ -713,9 +688,6 @@ def _handle_streaming(self, request, stream_id: str, response_sender) -> None: start = self._read_bool(request, "stream_start", False) end = self._read_bool(request, "stream_end", False) tokens = self._read_int_list(request, "text_token") - logger.info( - "CLIENT chunk: stream_id=%s start=%s end=%s tokens=%s", stream_id, start, end, tokens - ) if start: context_text = self._read_str(request, "context_text", self.default_context_text) @@ -731,7 +703,6 @@ def _handle_streaming(self, request, stream_id: str, response_sender) -> None: ) with self._sessions_lock: self._sessions[stream_id] = session - logger.info("rid=%s STREAM: opened session for stream_id=%s", session.request_id, stream_id) # All audio for the stream is sent on this (stream_start) sender by the # coroutine; do NOT complete it here. self._launch(self._synthesize_streaming(session)) From bc041bc3dc0da30d4d44fc23b2c054a2cc8b6440 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimkov Date: Wed, 10 Jun 2026 15:19:23 +0200 Subject: [PATCH 34/45] examples/tts/easymagpie_vllm_omni: fixes to docker file so it runs properly Signed-off-by: Viacheslav Klimkov --- examples/tts/easymagpie_vllm_omni/Dockerfile | 7 ++++++- examples/tts/easymagpie_vllm_omni/README.md | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/examples/tts/easymagpie_vllm_omni/Dockerfile b/examples/tts/easymagpie_vllm_omni/Dockerfile index e1047c2ec77c..52aa4d2ba1c1 100644 --- a/examples/tts/easymagpie_vllm_omni/Dockerfile +++ b/examples/tts/easymagpie_vllm_omni/Dockerfile @@ -25,7 +25,12 @@ RUN pip install --no-cache-dir \ onnx-graphsurgeon \ "tritonclient[grpc]" # have to force numpy after previous installations otherwise vllm_omni doesnt work -RUN pip install --no-cache-dir "numpy==2.3.5" +RUN pip uninstall -y numpy || true +RUN rm -rf \ + /usr/local/lib/python3.12/dist-packages/numpy \ + /usr/local/lib/python3.12/dist-packages/numpy.libs \ + /usr/local/lib/python3.12/dist-packages/numpy-*.dist-info +RUN pip install --no-cache-dir numpy==2.3.5 # 5. Restrict vLLM plugin auto-discovery to the EasyMagpie plugin only. ENV VLLM_PLUGINS=easymagpie_omni diff --git a/examples/tts/easymagpie_vllm_omni/README.md b/examples/tts/easymagpie_vllm_omni/README.md index df7970a8687d..2aa447e21063 100644 --- a/examples/tts/easymagpie_vllm_omni/README.md +++ b/examples/tts/easymagpie_vllm_omni/README.md @@ -45,7 +45,7 @@ streaming requests. vllm-omni 0.21.0rc1 + this plugin). ```bash - docker build -t easymp-vllm-omni examples/tts/easymagpie_vllm_omni/ + docker build --network=host -t easymp-vllm-omni examples/tts/easymagpie_vllm_omni/ ``` 4. **Launch the container** with the workspace and a GPU mounted. From 6dfbbeb92880a118e28f4c18a529212889cf70ac Mon Sep 17 00:00:00 2001 From: Viacheslav Klimkov Date: Wed, 10 Jun 2026 17:17:16 +0200 Subject: [PATCH 35/45] examples/tts/easymagpie_vllm_omni: reduce initial audio chunk latency Lower the first service audio chunk to one frame based on local TTFA benchmarks, and record the measured codec/streaming investigation notes. --- .../easymagpie_vllm_omni/model_repository/easymp/config.pbtxt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/tts/easymagpie_vllm_omni/model_repository/easymp/config.pbtxt b/examples/tts/easymagpie_vllm_omni/model_repository/easymp/config.pbtxt index b5d58f6a0707..f076bd9a481b 100644 --- a/examples/tts/easymagpie_vllm_omni/model_repository/easymp/config.pbtxt +++ b/examples/tts/easymagpie_vllm_omni/model_repository/easymp/config.pbtxt @@ -122,7 +122,7 @@ parameters { } parameters { key: "first_chunk_frames" - value: { string_value: "2" } + value: { string_value: "1" } } # Audio (local-transformer) sampling, forwarded via additional_information. From 058a2402f94ba604026a3928888e18fddf99c22b Mon Sep 17 00:00:00 2001 From: Viacheslav Klimkov Date: Thu, 11 Jun 2026 14:49:58 +0200 Subject: [PATCH 36/45] examples/tts/easymagpie_vllm_omni: some optimizations Signed-off-by: Viacheslav Klimkov --- .../model_repository/codec/config.pbtxt | 5 ++++- .../model_repository/easymp/1/model.py | 13 ++++++++++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/examples/tts/easymagpie_vllm_omni/model_repository/codec/config.pbtxt b/examples/tts/easymagpie_vllm_omni/model_repository/codec/config.pbtxt index a575964c4dba..d24d073a2c43 100644 --- a/examples/tts/easymagpie_vllm_omni/model_repository/codec/config.pbtxt +++ b/examples/tts/easymagpie_vllm_omni/model_repository/codec/config.pbtxt @@ -20,8 +20,11 @@ output [ } ] +# max_queue_delay=0: form a batch immediately with whatever is queued. At low +# concurrency the old 1ms delay was pure added TTFA latency with no batching +# benefit; under load requests still pile up fast enough to batch. dynamic_batching { - max_queue_delay_microseconds: 1000 + max_queue_delay_microseconds: 500 preferred_batch_size: [ 32 ] } diff --git a/examples/tts/easymagpie_vllm_omni/model_repository/easymp/1/model.py b/examples/tts/easymagpie_vllm_omni/model_repository/easymp/1/model.py index 1d53b8375225..6e1e3cff0908 100644 --- a/examples/tts/easymagpie_vllm_omni/model_repository/easymp/1/model.py +++ b/examples/tts/easymagpie_vllm_omni/model_repository/easymp/1/model.py @@ -463,9 +463,16 @@ def emit_ready(cum_codes, real_count: int, final: bool) -> None: continue if cum is None or cum_now.shape[0] > cum.shape[0]: cum = cum_now - # Hold back the most recent decode row: the audio-EOS frame is always - # the last one and must not be vocoded. - real_avail = cum.shape[0] - head - 1 + # The audio-EOS frame is always the most recent decoded row, so it is + # the only row that might be EOS. Inspect just that row: if it is not + # EOS, vocode it immediately instead of unconditionally holding one row + # back. This removes a full decode step (~1 ITL) from TTFA and from + # every chunk boundary; the authoritative tail scan below still catches + # the real EOS row. + if bool((cum[-1] == self.audio_eos_id).any()): + real_avail = cum.shape[0] - head - 1 + else: + real_avail = cum.shape[0] - head if real_avail > sent: emit_ready(cum, real_avail, final=False) From 8ce7d86d9850d279ad97bac259731760ead96e7b Mon Sep 17 00:00:00 2001 From: Viacheslav Klimkov Date: Fri, 12 Jun 2026 14:02:59 +0200 Subject: [PATCH 37/45] examples/tts/easymagpie_vllm_omni: replace local-transformer FFN Conv1d with Linear The local transformer's feed-forward used kernel-1 Conv1d layers, forcing a [b,t,c]<->[b,c,t] transpose on entry/exit that torch.compile could not fuse away (showed up as transpose/convolution triton kernels in profiling). Switch to bias-free nn.Linear operating directly on [b,t,c]; the conv submodule attribute name is kept and load_weights squeezes the trailing singleton dim so existing checkpoints still load 1:1. Also cache the positional arange index to avoid re-running an embedding gather every autoregressive step. Benchmark (Nemotron-H, -n 32 -c 1 32 --max-new-tokens 64): c=32 ITL 45.6->26.4ms, req/s 10.54->11.82. --- .../easymagpie_vllm_omni/easymagpie.py | 5 +++ .../easymagpie_vllm_omni/local_transformer.py | 44 ++++++++++++------- 2 files changed, 33 insertions(+), 16 deletions(-) diff --git a/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/easymagpie.py b/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/easymagpie.py index 601d2b0c3465..c24d83903c74 100644 --- a/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/easymagpie.py +++ b/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/easymagpie.py @@ -1021,6 +1021,11 @@ def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: if target is None: logger.warning("EasyMagpieTTS: no parameter for checkpoint key %s -> %s", name, mapped) continue + # The local-transformer FFN ships as kernel-1 ``Conv1d`` weights + # (``[out, in, 1]``) but now lives as ``nn.Linear`` (``[out, in]``). + # Squeeze the trailing singleton conv dim so the dense layer loads 1:1. + if tensor.ndim == target.ndim + 1 and tensor.shape[-1] == 1: + tensor = tensor.squeeze(-1) if target.shape != tensor.shape: raise RuntimeError( f"EasyMagpieTTS weight shape mismatch at {mapped!r}: " diff --git a/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/local_transformer.py b/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/local_transformer.py index aab5fe2f224b..4c2a0e89bb25 100644 --- a/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/local_transformer.py +++ b/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/local_transformer.py @@ -127,32 +127,39 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: class EasyMagpieLTFeedForward(nn.Module): """Positionwise feed-forward network. - Uses ``Conv1d(kernel_size=1)`` layers named ``proj.conv`` and ``o_net.conv`` - (no bias). A kernel-1 conv is a plain linear over the channel dim, applied - with a single transpose and GELU(tanh) in between. The ``Conv1d`` submodule - names match the training checkpoint so weights load 1:1. + A ``Conv1d(kernel_size=1)`` over the channel dim is mathematically identical + to an ``nn.Linear`` applied on the last dim, but the conv form forces a + ``[b, t, c] -> [b, c, t]`` transpose on the way in and out (which torch + cannot fuse away and which showed up as ``*_transpose_*`` / + ``*_convolution_*`` triton kernels in profiling). We therefore use plain + bias-free ``nn.Linear`` layers and operate directly on the ``[b, t, c]`` + layout. The ``conv`` submodule attribute is kept so the kernel-1 conv + weights from the training checkpoint (shape ``[out, in, 1]``) still map 1:1; + :meth:`EasyMagpieTTS.load_weights` squeezes the trailing singleton dim. """ def __init__(self, d_model: int, d_ffn: int) -> None: super().__init__() - self.proj = _Conv1dWrapper(d_model, d_ffn) - self.o_net = _Conv1dWrapper(d_ffn, d_model) + self.proj = _LinearWrapper(d_model, d_ffn) + self.o_net = _LinearWrapper(d_ffn, d_model) self.act = nn.GELU(approximate="tanh") def forward(self, x: torch.Tensor) -> torch.Tensor: - # x: [b, t, c] -> conv expects [b, c, t] - h = x.transpose(1, 2) - h = self.act(self.proj(h)) - h = self.o_net(h) - return h.transpose(1, 2) + # x: [b, t, c]; no transpose needed for a kernel-1 conv == linear. + return self.o_net(self.act(self.proj(x))) -class _Conv1dWrapper(nn.Module): - """Holds a kernel-1 ``Conv1d`` under attribute name ``conv`` (no bias).""" +class _LinearWrapper(nn.Module): + """Holds a bias-free ``nn.Linear`` under attribute name ``conv``. + + The attribute is named ``conv`` purely so the parameter path matches the + training checkpoint's kernel-1 ``Conv1d`` (``...proj.conv.weight``); the math + is a plain dense projection on the channel dim. + """ def __init__(self, in_ch: int, out_ch: int) -> None: super().__init__() - self.conv = nn.Conv1d(in_ch, out_ch, kernel_size=1, bias=False) + self.conv = nn.Linear(in_ch, out_ch, bias=False) def forward(self, x: torch.Tensor) -> torch.Tensor: return self.conv(x) @@ -206,6 +213,11 @@ def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: max_len = arch.num_stacked_codebooks + 2 self.position_embeddings = nn.Embedding(max_len, d_model) + # Cache a constant ``arange`` so we don't re-materialize it (and re-run an + # embedding gather) on every autoregressive step. The positional table is + # tiny and fixed; gathering once per forward over a cached index avoids + # the ``arange + embedding`` triton kernel seen in profiling. + self.register_buffer("_positions", torch.arange(max_len), persistent=False) self.layers = nn.ModuleList( [EasyMagpieLTLayer(d_model, d_ffn, n_heads) for _ in range(n_layers)] ) @@ -213,8 +225,8 @@ def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: def forward(self, inputs_embeds: torch.Tensor) -> torch.Tensor: seq_len = inputs_embeds.shape[1] - positions = torch.arange(seq_len, device=inputs_embeds.device) - x = inputs_embeds + self.position_embeddings(positions).unsqueeze(0) + pos_emb = self.position_embeddings(self._positions[:seq_len]) + x = inputs_embeds + pos_emb.unsqueeze(0) for layer in self.layers: x = layer(x) return self.norm_out(x) From 8828a93b8b64a50bfaa294d3e147eeb40fcf1b9e Mon Sep 17 00:00:00 2001 From: Viacheslav Klimkov Date: Fri, 12 Jun 2026 14:38:29 +0200 Subject: [PATCH 38/45] examples/tts/easymagpie_vllm_omni: capture local-transformer AR loop in a single graph The per-frame codebook loop replayed the compiled transformer N times with eager projection/sampling in between. Move the whole loop (transformer stack + per-codebook heads + sampling) into one @support_torch_compile module (EasyMagpieCodeLoop) so vLLM captures a single CUDA graph replayed once per frame instead of N times. Same FLOPs; removes per-step Python/launch overhead that dominates throughput scaling. Sampling is kept graph-safe: Gumbel noise is drawn eagerly outside the graph and injected (so RNG isn't replayed), temperature is a runtime tensor (per-request temperature still works), and top_k is a capture-time constant. The loop owns no params \u2014 the heads/embeddings/mask stay on EasyMagpieCodePredictor so the checkpoint still loads 1:1. Also squeeze the kernel-1 Conv1d->Linear weight in the test's NeMo->vLLM copy (follow-up to the FFN dense change). Benchmark (Nemotron-H, -n 32 -c 1 32 --max-new-tokens 64): c=32 req/s 11.82->17.55. --- .../easymagpie_vllm_omni/local_transformer.py | 230 +++++++++++------- .../tests/test_local_transformer.py | 12 +- 2 files changed, 146 insertions(+), 96 deletions(-) diff --git a/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/local_transformer.py b/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/local_transformer.py index 4c2a0e89bb25..3633b9902c18 100644 --- a/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/local_transformer.py +++ b/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/local_transformer.py @@ -43,51 +43,15 @@ from easymagpie_vllm_omni.config import EasyMagpieOmniArch -def _gumbel_argmax(logits: torch.Tensor) -> torch.Tensor: - """Gumbel-max categorical draw — CUDA-graph safe. +# Default top-k width for audio-codebook sampling. Because ``torch.topk``'s ``k`` +# shapes tensors inside the captured graph, this becomes a capture-time constant. +_DEFAULT_TOP_K = 80 - Equivalent to sampling from ``softmax(logits)`` but uses only - ``uniform_`` + ``log`` + ``argmax`` (all legal inside a captured graph) - and degrades gracefully on degenerate warmup logits instead of triggering - a device-side assert the way ``multinomial`` does. - """ - u = torch.empty_like(logits).uniform_(1e-20, 1.0 - 1e-20) - return (logits - torch.log(-torch.log(u))).argmax(dim=-1) - - -def sample_codebook( - logits: torch.Tensor, - *, - temperature: float, - top_k: int, - forbidden_mask: torch.Tensor | None, -) -> torch.Tensor: - """Sample one codebook's tokens from logits (CUDA-graph safe). - - Args: - logits: ``[num_tokens, vocab]`` raw codebook logits. - temperature: Sampling temperature; ``<= 0`` falls back to argmax. - top_k: Top-k truncation width (``<= 0`` disables truncation). - forbidden_mask: Optional ``[vocab]`` bool mask; ``True`` entries are - set to ``-inf`` before sampling (reserved/special tokens). - - Returns: - ``[num_tokens]`` int64 sampled token ids. - """ - if forbidden_mask is not None: - logits = logits.masked_fill(forbidden_mask, float("-inf")) - - if temperature <= 0.0: - return logits.argmax(dim=-1) - - logits = logits / temperature - - if top_k is not None and top_k > 0: - vals, idxs = torch.topk(logits, k=min(top_k, logits.size(-1)), dim=-1) - sampled_in_k = _gumbel_argmax(vals) - return idxs.gather(-1, sampled_in_k.unsqueeze(-1)).squeeze(-1) - - return _gumbel_argmax(logits) +# Minimum sampling temperature used inside the compiled graph. The old eager +# sampler special-cased ``temperature <= 0`` as exact argmax, but a +# data-dependent branch is illegal inside a captured graph, so we clamp to a tiny +# value (near-argmax) and always take the Gumbel-top-k path. +_MIN_SAMPLING_TEMPERATURE = 1e-4 class EasyMagpieLTSelfAttention(nn.Module): @@ -185,21 +149,14 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: return x -# NOTE: ``dynamic_arg_dims`` is passed explicitly rather than relying on -# vLLM's annotation-based inference. This file uses -# ``from __future__ import annotations`` (PEP 563), so ``forward``'s -# annotations are stored as strings (``"torch.Tensor"``) and vLLM's -# ``v.annotation in [torch.Tensor, ...]`` check would never match, raising -# "No dynamic dimensions found...". ``inputs_embeds`` is -# ``[num_tokens, num_codebooks, hidden]`` -> dim 0 (num_tokens) is dynamic. -@support_torch_compile(dynamic_arg_dims={"inputs_embeds": 0}) class EasyMagpieLocalTransformer(nn.Module): - """Compiled causal transformer stack with learnable positional embeddings. + """Causal transformer stack with learnable positional embeddings. - Decorated with ``@support_torch_compile`` so vLLM captures a single CUDA - graph for the fixed ``(num_tokens, num_stacked_codebooks, d_model)`` input - shape. Holds learnable ``position_embeddings``, the stacked ``layers.{i}.*`` - and a no-op ``norm_out``. + Plain (uncompiled) module: it is invoked from inside + :class:`EasyMagpieCodeLoop`'s compiled forward, so it gets *inlined* into + that single captured graph rather than being compiled / replayed on its own. + Holds learnable ``position_embeddings``, the stacked ``layers.{i}.*`` and a + no-op ``norm_out`` (names match the training checkpoint). """ def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: @@ -232,6 +189,95 @@ def forward(self, inputs_embeds: torch.Tensor) -> torch.Tensor: return self.norm_out(x) +# NOTE: ``dynamic_arg_dims`` is passed explicitly rather than relying on vLLM's +# annotation-based inference. This file uses ``from __future__ import +# annotations`` (PEP 563), so ``forward``'s annotations are stored as strings +# (``"torch.Tensor"``) and vLLM's ``v.annotation in [torch.Tensor, ...]`` check +# would never match, raising "No dynamic dimensions found...". Both ``dec_hidden`` +# and ``gumbel_noise`` are ``[num_tokens, ...]`` -> dim 0 (num_tokens) is dynamic. +@support_torch_compile(dynamic_arg_dims={"dec_hidden": 0, "gumbel_noise": 0}) +class EasyMagpieCodeLoop(nn.Module): + """Compiled single-graph autoregressive codebook loop. + + Runs the *entire* per-frame loop — transformer stack, per-codebook projection + heads, and (graph-safe) sampling — under one ``@support_torch_compile`` graph, + so vLLM captures a single CUDA graph replayed once per frame instead of + replaying the transformer ``N`` times with eager projection / sampling in + between. (Total FLOPs are unchanged — this removes per-step Python and + kernel-launch overhead, which dominates at low concurrency.) + + It owns **no parameters**: the projection / embedding / out-projection modules + and the forbidden mask live on the parent :class:`EasyMagpieCodePredictor` (so + the checkpoint still loads 1:1) and are reached through a non-registered + reference set by :meth:`bind_predictor`. + + Sampling is kept graph-safe by construction: + + * the Gumbel noise is drawn eagerly *outside* the graph and injected as + ``gumbel_noise`` — running ``uniform_`` inside the capture would otherwise + reuse the captured random numbers on every replay; + * ``temperature`` is a runtime tensor, so per-request temperature works + without recompiling; + * ``top_k`` shapes the ``topk`` / noise tensors and is therefore a + **capture-time constant** (per-request ``top_k`` changes are not honored + once the graph is captured). + """ + + def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: + super().__init__() + arch = EasyMagpieOmniArch.from_hf_config(vllm_config.model_config.hf_config) + self.num_codebooks = arch.num_stacked_codebooks + self.lt_hidden = arch.local_transformer_hidden_dim + self.top_k = min(_DEFAULT_TOP_K, arch.num_all_tokens_per_codebook) + # Set by :meth:`bind_predictor`; held in a tuple so nn.Module does not + # register the parent as a submodule (which would duplicate params). + self._predictor_ref: tuple = () + + def bind_predictor(self, predictor: "EasyMagpieCodePredictor") -> None: + self._predictor_ref = (predictor,) + self.top_k = predictor._sample_top_k + + def forward( + self, + dec_hidden: torch.Tensor, + gumbel_noise: torch.Tensor, + temperature: torch.Tensor, + ) -> torch.Tensor: + """Sample all ``num_codebooks`` codes for every frame in one graph. + + Args: + dec_hidden: ``[num_tokens, embedding_dim]`` backbone hidden state. + gumbel_noise: ``[num_tokens, num_codebooks, top_k]`` pre-drawn + Gumbel noise (``-log(-log(u))``), one slice per codebook. + temperature: ``[1]`` sampling temperature (already clamped > 0). + + Returns: + ``[num_tokens, num_codebooks]`` int64 sampled codes. + """ + cp = self._predictor_ref[0] + num_tokens = dec_hidden.shape[0] + n = self.num_codebooks + + buf = dec_hidden.new_zeros(num_tokens, n, self.lt_hidden) + buf[:, 0, :] = cp.local_transformer_in_projection(dec_hidden) + + forbidden = cp.forbidden_mask + codes: list[torch.Tensor] = [] + for k in range(n): + hidden = cp.local_transformer(buf) + row = cp.local_transformer_audio_out_projection(hidden[:, k, :]) + logits = cp.local_transformer_out_projections[k](row) + logits = logits.masked_fill(forbidden, float("-inf")) / temperature + vals, idxs = torch.topk(logits, self.top_k, dim=-1) + picked = (vals + gumbel_noise[:, k, :]).argmax(dim=-1, keepdim=True) + code_k = idxs.gather(-1, picked).squeeze(-1) + codes.append(code_k) + if k + 1 < n: + emb = cp.audio_in_projection(cp.audio_embeddings[k](code_k)) + buf[:, k + 1, :] = cp.local_transformer_in_projection(emb) + return torch.stack(codes, dim=1) + + class EasyMagpieCodePredictor(nn.Module): """Autoregressive intra-frame codebook predictor (the "local transformer"). @@ -304,15 +350,30 @@ def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: persistent=False, ) - # Sampling knobs (overridable from the outer model / request). + # Sampling knobs (overridable from the outer model / request). ``top_k`` + # is captured into the compiled loop graph (see ``EasyMagpieCodeLoop``), + # so per-request ``top_k`` changes are not honored once captured; + # per-request ``temperature`` is, since it is fed as a runtime tensor. self.temperature: float = 0.7 - self.top_k: int = 80 + self.top_k: int = _DEFAULT_TOP_K + self.lt_hidden = lt_hidden + self._sample_top_k = min(self.top_k, self.num_tokens_per_codebook) + + # Compiled single-graph autoregressive loop (owns no params; reaches the + # projection heads / embeddings / mask on ``self`` via a bound reference). + self._code_loop = EasyMagpieCodeLoop(vllm_config=vllm_config, prefix=f"{prefix}.code_loop") + self._code_loop.bind_predictor(self) # ── Persistent address-stable scratch buffers ────────────────── + # (created on the CUDA default device that vLLM sets during model init). max_num_tokens = vllm_config.scheduler_config.max_num_batched_tokens dtype = vllm_config.model_config.dtype - self._buf_inputs = torch.zeros(max_num_tokens, self.num_codebooks, lt_hidden, dtype=dtype) - self._out_codes = torch.zeros(max_num_tokens, self.num_codebooks, dtype=torch.long) + # Stable-address input for the captured loop graph. + self._dec_hidden_buf = torch.zeros(max_num_tokens, self.embedding_dim, dtype=dtype) + # Gumbel noise drawn eagerly each frame and injected into the graph; fp32 + # so the small ``-log(-log(u))`` values don't underflow in fp16. + self._gumbel_buf = torch.zeros(max_num_tokens, self.num_codebooks, self._sample_top_k, dtype=torch.float32) + self._temperature_buf = torch.zeros(1, dtype=torch.float32) @torch.no_grad() def init_forbidden_mask(self) -> None: @@ -352,14 +413,15 @@ def embed_audio_frame(self, codes: torch.Tensor) -> torch.Tensor: acc = acc / self.num_codebooks return self.audio_in_projection(acc) - def forward(self, inputs_embeds: torch.Tensor) -> torch.Tensor: - """Run the compiled local transformer over the input buffer.""" - return self.local_transformer(inputs_embeds) - @torch.no_grad() def generate_codes(self, dec_hidden: torch.Tensor) -> torch.Tensor: """Autoregressively sample all ``C * S`` codebooks for each frame. + Draws this frame's Gumbel noise eagerly into a stable buffer (fresh + randomness per frame, outside the captured graph) and stages the inputs + at fixed addresses, then runs the whole loop as a single captured graph + via :class:`EasyMagpieCodeLoop`. + Args: dec_hidden: ``[num_tokens, hidden]`` backbone hidden state (one row per frame being decoded). @@ -368,31 +430,13 @@ def generate_codes(self, dec_hidden: torch.Tensor) -> torch.Tensor: ``[num_tokens, num_codebooks]`` int64 sampled codes. """ num_tokens = dec_hidden.shape[0] - buf = self._buf_inputs[:num_tokens] - out = self._out_codes[:num_tokens] - buf.zero_() - - # Row 0: projected backbone hidden state (the AR "prompt"). - buf[:, 0, :] = self.local_transformer_in_projection(dec_hidden) - - # Always pass the mask unconditionally. An all-False mask makes - # ``masked_fill`` a no-op, so there's no need to guard with - # ``forbidden_mask.any()`` — and that guard is a data-dependent - # host sync that is illegal during CUDA-graph capture. - forbidden = self.forbidden_mask - for k in range(self.num_codebooks): - hidden = self(buf) # compiled transformer over the fixed buffer - row = self.local_transformer_audio_out_projection(hidden[:, k, :]) - logits = self.local_transformer_out_projections[k](row) - code_k = sample_codebook( - logits, - temperature=self.temperature, - top_k=self.top_k, - forbidden_mask=forbidden, - ) - out[:, k] = code_k - if k + 1 < self.num_codebooks: - emb = self.audio_in_projection(self.audio_embeddings[k](code_k)) - buf[:, k + 1, :] = self.local_transformer_in_projection(emb) - - return out[:num_tokens] + in_buf = self._dec_hidden_buf[:num_tokens] + in_buf.copy_(dec_hidden) + + # ``-log(-log(u))`` Gumbel noise, computed in place in fp32. + noise = self._gumbel_buf[:num_tokens] + noise.uniform_(1e-20, 1.0 - 1e-20) + noise.log_().neg_().log_().neg_() + + self._temperature_buf.fill_(max(float(self.temperature), _MIN_SAMPLING_TEMPERATURE)) + return self._code_loop(in_buf, noise, self._temperature_buf) diff --git a/examples/tts/easymagpie_vllm_omni/tests/test_local_transformer.py b/examples/tts/easymagpie_vllm_omni/tests/test_local_transformer.py index 21f9b3f92d98..b290aa510ba4 100644 --- a/examples/tts/easymagpie_vllm_omni/tests/test_local_transformer.py +++ b/examples/tts/easymagpie_vllm_omni/tests/test_local_transformer.py @@ -134,7 +134,7 @@ def _vllm_teacher_forced_logits( """ num_tokens = dec_hidden.shape[0] n = cp.num_codebooks - lt_hidden = cp._buf_inputs.shape[-1] + lt_hidden = cp.lt_hidden buf = torch.zeros(num_tokens, n, lt_hidden, dtype=dec_hidden.dtype, device=dec_hidden.device) buf[:, 0, :] = cp.local_transformer_in_projection(dec_hidden) for k in range(n - 1): @@ -154,8 +154,14 @@ def _copy_nemo_into_vllm(nemo: NeMoLocalTransformerStack, cp: EasyMagpieCodePred missing = [] for name, param in cp.named_parameters(): if name in nemo_sd: - assert param.shape == nemo_sd[name].shape, f"shape mismatch {name}" - param.data.copy_(nemo_sd[name].to(param.dtype)) + src = nemo_sd[name] + # The FFN ships as kernel-1 Conv1d (``[out, in, 1]``) in NeMo but is a + # plain ``nn.Linear`` (``[out, in]``) here; squeeze the conv dim to + # match (mirrors ``EasyMagpieTTS.load_weights``). + if src.ndim == param.ndim + 1 and src.shape[-1] == 1: + src = src.squeeze(-1) + assert param.shape == src.shape, f"shape mismatch {name}" + param.data.copy_(src.to(param.dtype)) else: missing.append(name) assert not missing, f"vLLM params with no NeMo counterpart: {missing}" From ea9689a48cecbcd6cf79d2b361ea796b03bd14ca Mon Sep 17 00:00:00 2001 From: Viacheslav Klimkov Date: Fri, 12 Jun 2026 17:59:36 +0200 Subject: [PATCH 39/45] examples/tts/easymagpie_vllm_omni: optimize preprocess prefill path preprocess runs on the host, once per request, serially on the runner's critical path; shipping a per-request (T_audio, embedding_dim) speaker embedding (ZMQ serialize/deserialize + H2D) and reassembling the prefill context there dominated TTFT under concurrency. For the fixed speaker set we serve, bake the speaker embeddings into model state: load each speaker_embeddings/.pt once at construction into a GPU-resident, model-dtype tensor, so a request carries only a short speaker_id string. Custom / one-off voices may still pass a raw speaker_embedding tensor. Loaded in __init__ (not load_weights) so it is present under --load-format dummy too. Also drop the silent zero/last-row padding of short prefill chunks in favor of an assertion (the backbone was never trained on padded context). Benchmark (dummy weights, RTX A6000, n=64, speaker_id path): c=32 TTFT 188ms -> ~95ms, c=1 31ms -> 27ms. --- .../easymagpie_vllm_omni/easymagpie.py | 191 ++++++++++++++---- 1 file changed, 157 insertions(+), 34 deletions(-) diff --git a/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/easymagpie.py b/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/easymagpie.py index c24d83903c74..a7f9977059fd 100644 --- a/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/easymagpie.py +++ b/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/easymagpie.py @@ -44,9 +44,19 @@ Per-request I/O (via ``additional_information``): -* ``speaker_embedding`` (prefill only) — ``(T_audio, embedding_dim)`` - speaker-encoded context-audio embedding. ``preprocess`` assembles the full - prefill context embedding itself as +* speaker context audio (prefill only) — either of: + + * ``speaker_id`` — a short string naming a *known* speaker whose + ``(T_audio, embedding_dim)`` context-audio embedding is precomputed model + state: every ``speaker_embeddings/.pt`` in the checkpoint dir is loaded + once at construction into a GPU-resident, model-dtype tensor (see + :meth:`_load_speaker_embeddings`). Requests then carry just the id — no + per-request H2D and no serialization of a multi-hundred-KB tensor through the + engine. This is the recommended path for serving a fixed speaker set. + * ``speaker_embedding`` — a ``(T_audio, embedding_dim)`` tensor for custom / + one-off voices, copied H2D on each prefill. + + ``preprocess`` assembles the full prefill context embedding itself as ``[task_embedding | speaker_embedding | context_text_embedded]``, so the caller only does the speaker-encoder math and passes plain context text (the model tokenizes + embeds it and prepends the per-mode service token). @@ -239,6 +249,10 @@ def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: # caller passes plain text, never pre-tokenized ids. self._text_tokenizer: Any = None + # Known-speaker embedding table (populated at the end of ``__init__``; + # see :meth:`_load_speaker_embeddings`). + self._speaker_embeddings: dict[str, torch.Tensor] = {} + # ── Streaming delays (text leads phoneme by ``phonemes_delay`` and audio # by ``speech_delay`` decode steps; 0/0 == lock-step). ── self.phonemes_delay = int(getattr(arch, "streaming_phonemes_delay", 0) or 0) @@ -292,6 +306,20 @@ def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: # ``compute_logits`` self._sample_stop = torch.zeros(max_num_tokens, dtype=torch.bool) + # ── Known-speaker context-audio embeddings (precomputed model state) ── + # ``preprocess`` runs on the host, once per request, serially on the + # runner's critical path, so shipping a per-request + # ``(T_audio, embedding_dim)`` speaker tensor (ZMQ serialize/deserialize + + # H2D) dominates TTFT under concurrency. For the fixed speaker set we + # serve, bake those embeddings into model state instead: load each + # ``speaker_embeddings/.pt`` once into a GPU-resident, model-dtype + # tensor so a request only carries a short ``speaker_id`` string. Custom / + # one-off voices may still pass a raw ``speaker_embedding`` tensor (see + # :meth:`_resolve_speaker_embedding`). Loaded here (not in + # :meth:`load_weights`) so it is present even under ``--load-format dummy``, + # which skips weight loading. + self._speaker_embeddings = self._load_speaker_embeddings() + # ------------------------------------------------------------------ # Embedding helpers # ------------------------------------------------------------------ @@ -688,17 +716,19 @@ def _preprocess_prefill( offset = int(info_dict.get("prefill_offset", 0) or 0) total = int(prefill_embeds.shape[0]) - s = max(0, min(offset, total)) - e = max(0, min(offset + span_len, total)) - take = prefill_embeds[s:e] - if int(take.shape[0]) < span_len: - pad_n = span_len - int(take.shape[0]) - pad_rows = ( - take[-1:].expand(pad_n, -1) - if take.shape[0] > 0 - else prefill_embeds.new_zeros(pad_n, prefill_embeds.shape[-1]) - ) - take = torch.cat([take, pad_rows], dim=0) + take = prefill_embeds[offset : offset + span_len] + # The prefill chunk must lie fully within the assembled context. Padding + # short chunks with zeros / a repeated last row is invalid: the backbone + # was never trained on padded context frames, so silently doing so would + # corrupt conditioning rather than fail loudly. This holds iff the caller + # sized ``prompt_token_ids`` to ``estimate_prompt_len(...)``. + assert int(take.shape[0]) == span_len, ( + f"EasyMagpieTTS prefill chunk [{offset}:{offset + span_len}] is not fully covered by the " + f"assembled context embedding (length {total}). The caller must pass " + f"prompt_token_ids of length estimate_prompt_len(...) = " + f"[task?] + speaker_embedding.shape[0] + len(tokenize(context_text)); " + f"zero-padding the backbone context is invalid (the model was not trained on it)." + ) info_update = { "prefill_offset": offset + span_len, @@ -731,27 +761,19 @@ def _build_prefill_embeds( from the per-request inputs: - * ``speaker_embedding`` — the speaker-encoded context-audio embedding, - required as a 2-D ``(T_audio, embedding_dim)`` tensor. + * speaker context audio — either ``speaker_id`` (a known speaker whose + embedding is precomputed model state, see + :meth:`_resolve_speaker_embedding`) or, for custom / one-off voices, a + 2-D ``(T_audio, embedding_dim)`` ``speaker_embedding`` tensor. * ``context_text`` — a plain string (e.g. ``"[EN]"``); tokenized in-model - (see :meth:`_encode_context_text`) and embedded through the baked - per-subword ``text_embedding`` table. + and embedded through the baked per-subword ``text_embedding`` table. * ``task_mode_id`` — selects the per-mode task ("service token") embedding row; prepended only when the checkpoint has a task table. - Returns the full context embedding; the per-chunk slicing/padding is done - by :meth:`_preprocess_prefill`. + Returns the full context embedding; the per-chunk slicing is done by + :meth:`_preprocess_prefill`. """ dtype = self._combined_embeddings.dtype - - speaker_embedding = info_dict.get("speaker_embedding") - assert isinstance(speaker_embedding, torch.Tensor) and speaker_embedding.ndim == 2, ( - "EasyMagpieTTS preprocess expects additional_information.speaker_embedding to be a 2-D " - "(T_audio, embedding_dim) tensor (the speaker-encoded context audio); " - f"got {type(speaker_embedding).__name__}" - + (f" with ndim={speaker_embedding.ndim}" if isinstance(speaker_embedding, torch.Tensor) else "") - ) - parts: list[torch.Tensor] = [] # Task / "service token" embedding (prepended), when present. @@ -761,8 +783,8 @@ def _build_prefill_embeds( task_row = self.task_embedding(torch.tensor([task_mode_id], device=device, dtype=torch.long)) parts.append(task_row.to(dtype)) - # Speaker-encoded context audio. - parts.append(speaker_embedding.to(device=device, dtype=dtype)) + # Speaker-encoded context audio (known-speaker state or custom tensor). + parts.append(self._resolve_speaker_embedding(device, info_dict)) # Context text: tokenized in-model and embedded through the baked table. context_text = self._first_str(info_dict.get("context_text")) or _DEFAULT_CONTEXT_TEXT @@ -772,6 +794,69 @@ def _build_prefill_embeds( return torch.cat(parts, dim=0) + def _resolve_speaker_embedding(self, device: torch.device, info_dict: dict[str, Any]) -> torch.Tensor: + """Return the speaker context-audio embedding on ``device`` in model dtype. + + Prefers a known ``speaker_id`` resolved from precomputed model state (see + :meth:`_load_speaker_embeddings`) — no per-request transfer. Falls back to + a raw ``speaker_embedding`` tensor (custom / one-off voice), copied H2D + here. Exactly one of the two must be supplied. + """ + dtype = self._combined_embeddings.dtype + speaker_id = self._first_str(info_dict.get("speaker_id")) + if speaker_id: + embedding = self._speaker_embeddings.get(speaker_id) + assert embedding is not None, ( + f"EasyMagpieTTS preprocess got unknown speaker_id {speaker_id!r}; known speakers: " + f"{sorted(self._speaker_embeddings)}. Register it under the checkpoint's " + "speaker_embeddings/ dir, or pass a raw speaker_embedding tensor for a custom voice." + ) + # One-time device pin (no-op once resident) — the table is built in + # __init__, but guard in case the module was placed after construction. + if embedding.device != device: + embedding = embedding.to(device=device, dtype=dtype) + self._speaker_embeddings[speaker_id] = embedding + return embedding + + speaker_embedding = info_dict.get("speaker_embedding") + assert isinstance(speaker_embedding, torch.Tensor) and speaker_embedding.ndim == 2, ( + "EasyMagpieTTS preprocess expects additional_information.speaker_id (a known speaker) or " + "speaker_embedding as a 2-D (T_audio, embedding_dim) tensor (the speaker-encoded context " + f"audio); got speaker_embedding={type(speaker_embedding).__name__}" + + (f" with ndim={speaker_embedding.ndim}" if isinstance(speaker_embedding, torch.Tensor) else "") + ) + return speaker_embedding.to(device=device, dtype=dtype) + + def _load_speaker_embeddings(self) -> dict[str, torch.Tensor]: + """Load all known-speaker embeddings from ``/speaker_embeddings/*.pt``. + + Each file holds either a bare ``(T_audio, embedding_dim)`` tensor or a dict + with a ``speaker_encoding`` key (the converter/caller layout). They are + moved to the model's device + dtype once here so per-request prefill only + sources the speaker part from this table (no transfer). Returns an empty + table when the directory is absent (custom-voice-only deployments). + """ + import glob + import os + + out: dict[str, torch.Tensor] = {} + spk_dir = os.path.join(self.model_path, "speaker_embeddings") + if not os.path.isdir(spk_dir): + return out + device = self._combined_embeddings.device + dtype = self._combined_embeddings.dtype + for path in sorted(glob.glob(os.path.join(spk_dir, "*.pt"))): + name = os.path.splitext(os.path.basename(path))[0] + loaded = torch.load(path, map_location="cpu") + embedding = loaded["speaker_encoding"] if isinstance(loaded, dict) else loaded + if not (isinstance(embedding, torch.Tensor) and embedding.ndim == 2): + logger.warning("EasyMagpieTTS: skipping speaker embedding %s (expected a 2-D tensor)", path) + continue + out[name] = embedding.to(device=device, dtype=dtype) + if out: + logger.info("EasyMagpieTTS: loaded %d known speaker embedding(s): %s", len(out), ", ".join(sorted(out))) + return out + def _maybe_set_lt_sampling_params(self, info_dict: dict[str, Any]) -> None: """Apply per-request audio sampling params to the local transformer. @@ -832,13 +917,19 @@ def estimate_prompt_len( context_text: str = _DEFAULT_CONTEXT_TEXT, has_task_embedding: bool = False, ) -> int: - """Length-only mirror of :meth:`_build_prefill_embeds`. + """Length-only mirror of :meth:`_build_prefill_embeds` (custom voice). The engine assembles the prefill context as ``[task_embedding? | speaker_embedding | context_text_embedded]``, so the caller must pass ``prompt_token_ids = [0] * estimate_prompt_len(...)`` for the placeholder length to match the assembled embedding length (otherwise - vLLM pads / truncates and quality drops). + vLLM pads / truncates and quality drops). This is a pure function of + lengths, so it stays static — callable in the request-building process + without an engine instance. + + For a **known speaker** the caller holds only a ``speaker_id`` (not the + tensor); use :meth:`get_prompt_len`, which loads the embedding from the + checkpoint dir and calls this method. Args: speaker_embedding: ``(T_audio, embedding_dim)`` speaker-encoded @@ -846,7 +937,7 @@ def estimate_prompt_len( tokenize: callable turning ``context_text`` into its subword ids (e.g. ``lambda t: tokenizer.encode(t)``) — must match the tokenizer the engine loads from ``model_path``. - context_text: conditioning string (default ``"[NO TEXT CONTEXT]"``). + context_text: conditioning string (default ``"[EN]"``). has_task_embedding: whether the checkpoint prepends a task / "service token" embedding (``num_task_embeddings > 0``). """ @@ -855,6 +946,38 @@ def estimate_prompt_len( task_len = 1 if has_task_embedding else 0 return task_len + t_audio + ctx_len + @classmethod + def get_prompt_len(cls, speaker_id: str, model_path: str, *, tokenize: Callable[[str], Iterable[int]]) -> int: + """Known-speaker convenience wrapper around :meth:`estimate_prompt_len`. + + Resolves everything from the checkpoint dir so it cannot disagree with + what the engine actually uses: loads the speaker embedding from + ``speaker_embeddings/.pt`` (see :meth:`_load_speaker_embeddings`), + reads ``has_task_embedding`` from ``config.json`` (``num_task_embeddings``), + and conditions on the fixed :data:`_DEFAULT_CONTEXT_TEXT`. Lets a caller + holding only a ``speaker_id`` size ``prompt_token_ids`` without an engine + instance (``context_text`` / ``has_task_embedding`` are intentionally not + params — they must match the precomputed checkpoint, not be overridden). + """ + import json + import os + + path = os.path.join(model_path, "speaker_embeddings", f"{speaker_id}.pt") + if not os.path.exists(path): + raise FileNotFoundError(f"EasyMagpieTTS: no speaker embedding {path} for speaker_id {speaker_id!r}") + loaded = torch.load(path, map_location="cpu") + speaker_embedding = loaded["speaker_encoding"] if isinstance(loaded, dict) else loaded + + with open(os.path.join(model_path, "config.json")) as f: + num_task_embeddings = int(json.load(f).get("num_task_embeddings", 0)) + + return cls.estimate_prompt_len( + speaker_embedding, + tokenize=tokenize, + context_text=_DEFAULT_CONTEXT_TEXT, + has_task_embedding=num_task_embeddings > 0, + ) + def _preprocess_decode( self, input_ids: torch.Tensor, From a4237408d18fcede0827efad26f4ab85478b9832 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimkov Date: Mon, 15 Jun 2026 17:57:30 +0200 Subject: [PATCH 40/45] examples/tts/easymagpie_vllm_omni: fix service to work with speaker_id, no-op for codec as debug Signed-off-by: Viacheslav Klimkov --- .../model_repository/easymp/1/model.py | 252 +++++++++++------- .../model_repository/easymp/config.pbtxt | 9 + 2 files changed, 164 insertions(+), 97 deletions(-) diff --git a/examples/tts/easymagpie_vllm_omni/model_repository/easymp/1/model.py b/examples/tts/easymagpie_vllm_omni/model_repository/easymp/1/model.py index 6e1e3cff0908..1ea04514e312 100644 --- a/examples/tts/easymagpie_vllm_omni/model_repository/easymp/1/model.py +++ b/examples/tts/easymagpie_vllm_omni/model_repository/easymp/1/model.py @@ -69,6 +69,10 @@ # (``stream_end``): the input generator then appends text-EOS + the mask sentinel. _STREAM_END = object() +# Sentinel pushed onto a request's codec queue when the engine generator is fully +# drained (clean end): the codec worker flushes the trailing window as final. +_GEN_DONE = object() + class _StreamingSession: """Per-``stream_id`` state for a streaming-text request. @@ -106,6 +110,17 @@ def _require_param(parameters: dict, key: str) -> str: return str(val) +def _optional_param(parameters: dict, key: str, default: str) -> str: + val = parameters.get(key) + if isinstance(val, dict): + val = val.get("string_value") + return str(val) if val is not None else default + + +def _as_bool(val: str) -> bool: + return str(val).strip().lower() in ("1", "true", "yes", "on") + + class TritonPythonModel: def initialize(self, args): os.environ.setdefault("VLLM_WORKER_MULTIPROC_METHOD", "spawn") @@ -130,9 +145,17 @@ def initialize(self, args): self.lt_temperature = float(_require_param(params, "lt_temperature")) self.lt_top_k = int(_require_param(params, "lt_top_k")) + # Benchmark toggle: when set, skip the codec BLS entirely (no GPU->CPU copy, + # no codec inference, no audio) so the AR/orchestration path in this model can + # be measured in isolation against benchmark_model.py. Returns silence chunks. + self.codec_noop = _as_bool(_optional_param(params, "codec_noop", "false")) + # Samples emitted per model frame in codec_noop mode, used only to size the + # silence chunks. One model frame = 2 codec frames @ 12.5 fps -> 24000/12.5 is + # the codec rate; here 22050/12.5 = 1764 samples per model frame. + self.codec_noop_spf = int(_optional_param(params, "codec_noop_spf", "1764")) + self._load_arch_and_tokenizer() self._init_sampling_helpers() - self._speaker_cache: dict = {} # Inferred from the first codec decode (audio_len / codec_chunk_size). self._spf: int | None = None @@ -155,7 +178,13 @@ def initialize(self, args): ) self._start_omni_engine() - logger.info("EasyMagpie initialized (default_speaker=%s)", self.default_speaker) + logger.info( + "EasyMagpie initialized (default_speaker=%s, codec_noop=%s)", + self.default_speaker, + self.codec_noop, + ) + if self.codec_noop: + logger.warning("codec_noop=True: codec decode is DISABLED; responses carry silence (benchmark mode).") def _load_arch_and_tokenizer(self): from transformers import AutoTokenizer @@ -170,14 +199,13 @@ def _load_arch_and_tokenizer(self): self.audio_eos_id = int(arch.audio_eos_id) self.speech_delay = int(getattr(arch, "streaming_speech_delay", 0) or 0) self.num_stacked_codebooks = int(arch.num_stacked_codebooks) - self.has_task_embedding = arch.num_task_embeddings > 0 self.stop_token_id = EasyMagpieTTSForConditionalGeneration.audio_eos_stop_token_id(cfg_obj) # Appended after the client's streamed subword ids to close the text channel # before the free-running acoustic tail (matches the demo / benchmark). self.text_eos_id = int(config.get("text_vocab_size", config.get("vocab_size", 0))) - 2 self.tokenizer = AutoTokenizer.from_pretrained(self.vllm_model_path, trust_remote_code=True) - self._estimate_prompt_len = EasyMagpieTTSForConditionalGeneration.estimate_prompt_len + self._get_prompt_len = EasyMagpieTTSForConditionalGeneration.get_prompt_len def _init_sampling_helpers(self): """Cache the vLLM sampling/streaming types used by both request flavours.""" @@ -280,33 +308,28 @@ def _start_omni_engine(self): stage_init_timeout=300, ) - def _get_speaker_embedding(self, speaker: str) -> torch.Tensor: - if speaker not in self._speaker_cache: - emb_path = Path(self.vllm_model_path) / "speaker_embeddings" / f"{speaker}.pt" - if not emb_path.exists(): - raise FileNotFoundError(f"Speaker embedding not found: {emb_path}") - loaded = torch.load(emb_path, map_location="cpu") - emb = loaded["speaker_encoding"] if isinstance(loaded, dict) else loaded - self._speaker_cache[speaker] = emb.to(torch.float32) - return self._speaker_cache[speaker] - - def _prompt_len(self, speaker_embedding: torch.Tensor, context_text: str) -> int: + def _prompt_len(self, speaker: str) -> int: + """Placeholder ``prompt_token_ids`` length for a known speaker. + + Resolves everything from the checkpoint dir (speaker embedding + + ``has_task_embedding``), so the caller only holds a ``speaker_id`` and + never loads / ships the embedding itself (the engine sources it from its + precomputed speaker table via ``speaker_id``). + """ return int( - self._estimate_prompt_len( - speaker_embedding, + self._get_prompt_len( + speaker, + self.vllm_model_path, tokenize=lambda t: self.tokenizer.encode(t), - context_text=context_text, - has_task_embedding=self.has_task_embedding, ) ) def _build_prompt(self, text: str, context_text: str, speaker: str) -> dict: - speaker_embedding = self._get_speaker_embedding(speaker) - prompt_len = self._prompt_len(speaker_embedding, context_text) + prompt_len = self._prompt_len(speaker) return { "prompt_token_ids": [0] * prompt_len, "additional_information": { - "speaker_embedding": speaker_embedding, + "speaker_id": speaker, "context_text": context_text, "text": text, "temperature": self.lt_temperature, @@ -316,7 +339,17 @@ def _build_prompt(self, text: str, context_text: str, speaker: str) -> dict: def _decode_codec(self, codes: torch.Tensor, left_context_frames: int) -> np.ndarray: """Decode one ``(<=codec_chunk_size, C*S)`` window, trim left context + pad.""" - codes_np = codes.detach().cpu().to(torch.int64).numpy() + if self.codec_noop: + # Benchmark mode: skip the GPU->CPU copy and codec BLS entirely; return + # silence sized to the real (post-left-context) frames so the streaming + # cadence and response sizes stay roughly representative. + spf = self._spf or self.codec_noop_spf + n_frames = max(0, int(codes.shape[0]) - int(left_context_frames)) + return np.zeros(n_frames * spf, dtype=np.float32) + + # Cast on the host (codec wants int64): the window is tiny so this is cheap, + # and it avoids an extra GPU cast kernel + a wider device->host copy. + codes_np = codes.detach().cpu().numpy().astype(np.int64, copy=False) pad = self.codec_chunk_size - codes_np.shape[0] if pad > 0: codes_np = np.pad(codes_np, ((0, pad), (0, 0))) @@ -359,23 +392,82 @@ def _send_error(self, response_sender, err: Exception): except Exception: pass - def _codec_worker(self, codec_q: queue.Queue, response_sender, state: dict) -> None: - """Pop ``(chunk, ctx, is_final)`` tuples; ``None`` == send empty final + exit.""" + def _codec_worker(self, codec_q: queue.Queue, response_sender, state: dict, head: int) -> None: + """Per-request codec pump: runs the whole accumulate -> chunk -> decode -> send + pipeline on a pool thread so the shared asyncio loop does no per-step tensor + work at all. + + Queue protocol (pushed by :meth:`_drive_codec`): + * ``(cum_codes, hit_eos)`` — the latest cumulative ``(T, C*S)`` codes tensor + and whether the backbone audio-EOS stop token fired on that step (a cheap + CPU flag, never a tensor scan). We keep the largest cumulative seen. + * ``_GEN_DONE`` — the engine generator drained cleanly; flush the tail final. + * ``None`` — error/abort; send an empty final and exit. + + Rows ``[0, head)`` are prefill + ``speech_delay`` warm-up; the first real audio + frame is at ``head``. When the EOS step is seen its last row is the audio-EOS + frame, so we never vocode it. + """ + L = self.codec_left_context + sent = 0 # real frames already vocoded + sent + threshold = self.first_chunk_frames + cum = None # largest cumulative codes tensor seen + saw_eos = False finalized = False - try: - while True: - item = codec_q.get() - if item is None: - self._send_audio(response_sender, np.array([], dtype=np.float32), final=True) - finalized = True - return - chunk, ctx, is_final = item + + def emit_ready(real_count: int, final: bool) -> None: + """Vocode + send overlap-save windows of newly-ready real frames.""" + nonlocal sent, threshold, finalized + while sent < real_count: + remaining = real_count - sent + if not final and remaining < threshold: + break + take = min(threshold, remaining) + ctx = min(sent, L) + chunk = cum[head + sent - ctx : head + sent + take] + sent += take + threshold = self.codec_chunk_size - L + is_final = final and sent >= real_count audio = self._decode_codec(chunk, ctx) self._send_audio(response_sender, audio, final=is_final) if state["t_first_audio"] is None: state["t_first_audio"] = time.perf_counter() - if is_final: - finalized = True + finalized = finalized or is_final + + try: + done = False + while True: + # Block for the next item, then drain any backlog so we only act on the + # most recent cumulative codes (older snapshots are subsets of it). + batch = [codec_q.get()] + while True: + try: + batch.append(codec_q.get_nowait()) + except queue.Empty: + break + for item in batch: + if item is _GEN_DONE: + done = True + elif item is None: + self._send_audio(response_sender, np.array([], dtype=np.float32), final=True) + finalized = True + return + else: + cum_now, hit_eos = item + if cum is None or cum_now.shape[0] > cum.shape[0]: + cum = cum_now + saw_eos = saw_eos or hit_eos + + if state["error"] is not None: + return + if cum is not None: + real_avail = max(0, cum.shape[0] - head - (1 if saw_eos else 0)) + if done or real_avail > sent: + emit_ready(real_avail, final=done) + if done: + if not finalized: + self._send_audio(response_sender, np.array([], dtype=np.float32), final=True) + finalized = True return except Exception as e: state["error"] = e @@ -408,10 +500,19 @@ async def _drive_codec( streaming-text); from here on both flavours are identical. All audio is sent on ``response_sender`` (for streaming that is the ``stream_start`` sender). + This coroutine stays deliberately thin: per step it only reads the cumulative + ``audio_codes`` reference and a cheap CPU end-of-stream flag, then hands both + to the per-request :meth:`_codec_worker` thread. All tensor slicing, the + device->host copy, codec inference and ``response_sender.send`` happen on that + pool thread, so the single shared event loop never does per-step tensor work + (and never blocks on a GPU sync) — that is what lets many requests share the + loop without serializing. + Each step yields the *cumulative* codes ``(prompt_len prefill rows + decoded frames, C*S)`` (as a list to cat or an already-consolidated tensor); the first - real audio frame is at row ``prompt_len + speech_delay`` and the last decoded - row is the audio-EOS frame. + real audio frame is at row ``prompt_len + speech_delay``. End of stream is the + backbone audio-EOS stop token, surfaced as ``outputs[0].stop_reason`` (a CPU + attribute) — no per-step scan of the codes tensor is needed. For streaming-text, ``pace_q`` carries one token per observed decode-step output so the input feeder releases the next chunk only after the previous one @@ -421,33 +522,9 @@ async def _drive_codec( codec_q: queue.Queue = queue.Queue() state: dict = {"t_first_audio": None, "error": None} - codec_future = self._codec_pool.submit(self._codec_worker, codec_q, response_sender, state) - - # Codes are a cumulative ``(T, C*S)`` tensor: rows [0, prompt_len) are the - # prefill prefix, the next ``speech_delay`` rows are warm-up, so the first - # real audio frame is at ``head`` and the last decoded row is audio-EOS. - L = self.codec_left_context head = prompt_len + self.speech_delay # cumulative row of the first real frame - sent = 0 # real frames already queued to the codec - threshold = self.first_chunk_frames - cum = None # largest cumulative codes tensor seen - produced_final = False - - def emit_ready(cum_codes, real_count: int, final: bool) -> None: - """Queue overlap-save windows of newly-ready real frames (by cum row).""" - nonlocal sent, threshold, produced_final - while sent < real_count: - remaining = real_count - sent - if not final and remaining < threshold: - break - take = min(threshold, remaining) - ctx = min(sent, L) - chunk = cum_codes[head + sent - ctx : head + sent + take] - sent += take - threshold = self.codec_chunk_size - L - is_final = final and sent >= real_count - codec_q.put((chunk, ctx, is_final)) - produced_final = produced_final or is_final + codec_future = self._codec_pool.submit(self._codec_worker, codec_q, response_sender, state, head) + sent = 0 # forwarded steps (for the log line only) try: async for out in gen: @@ -461,35 +538,17 @@ def emit_ready(cum_codes, real_count: int, final: bool) -> None: cum_now = self._cumulative_codes(payload) if cum_now is None: continue - if cum is None or cum_now.shape[0] > cum.shape[0]: - cum = cum_now - # The audio-EOS frame is always the most recent decoded row, so it is - # the only row that might be EOS. Inspect just that row: if it is not - # EOS, vocode it immediately instead of unconditionally holding one row - # back. This removes a full decode step (~1 ITL) from TTFA and from - # every chunk boundary; the authoritative tail scan below still catches - # the real EOS row. - if bool((cum[-1] == self.audio_eos_id).any()): - real_avail = cum.shape[0] - head - 1 - else: - real_avail = cum.shape[0] - head - if real_avail > sent: - emit_ready(cum, real_avail, final=False) - - if state["error"] is None and cum is not None: - # Authoritative tail: scan for the audio-EOS row (only it carries - # audio_eos_id > codebook_size) and vocode every real frame before it. - eos_row = None - for i in range(cum.shape[0] - 1, head - 1, -1): - if bool((cum[i] == self.audio_eos_id).any()): - eos_row = i - break - last_excl = eos_row if eos_row is not None else cum.shape[0] - real_count = max(0, last_excl - head) - emit_ready(cum, real_count, final=True) - if not produced_final: - codec_q.put(None) - elif state["error"] is None: + # The request truly ends at the backbone audio-EOS stop token; vLLM + # already detected it and reports it as the matched stop_reason. When it + # fires this step's last row is the EOS frame (which the worker drops). + co = out.outputs[0] if getattr(out, "outputs", None) else None + hit_eos = getattr(co, "stop_reason", None) == self.stop_token_id + sent += 1 + codec_q.put((cum_now, hit_eos)) + + if state["error"] is None: + codec_q.put(_GEN_DONE) + else: codec_q.put(None) await asyncio.wrap_future(codec_future) @@ -499,7 +558,7 @@ def emit_ready(cum_codes, real_count: int, final: bool) -> None: t_end = time.perf_counter() ttfa_ms = ((state["t_first_audio"] or t_end) - t_start) * 1000 logger.info( - "rid=%s ttfa=%.1fms total=%.1fms frames=%d speaker=%s text=%r", + "rid=%s ttfa=%.1fms total=%.1fms steps=%d speaker=%s text=%r", request_id, ttfa_ms, (t_end - t_start) * 1000, @@ -557,12 +616,11 @@ async def _stream_inputs(self, session: _StreamingSession): client actually having sent them. """ StreamingInput = self._StreamingInput - speaker_embedding = self._get_speaker_embedding(session.speaker) - prompt_len = self._prompt_len(speaker_embedding, session.context_text) + prompt_len = self._prompt_len(session.speaker) sp1 = self._make_sampling_params(1) prefill_info = { - "speaker_embedding": speaker_embedding, + "speaker_id": session.speaker, "context_text": session.context_text, "temperature": self.lt_temperature, "top_k": self.lt_top_k, @@ -602,7 +660,7 @@ async def _stream_inputs(self, session: _StreamingSession): yield self._text_chunk(-1, tail_params) async def _synthesize_streaming(self, session: _StreamingSession): - prompt_len = self._prompt_len(self._get_speaker_embedding(session.speaker), session.context_text) + prompt_len = self._prompt_len(session.speaker) inputs_gen = self._stream_inputs(session) gen = self.omni.generate( inputs_gen, sampling_params_list=[self._make_sampling_params(1)], request_id=session.request_id diff --git a/examples/tts/easymagpie_vllm_omni/model_repository/easymp/config.pbtxt b/examples/tts/easymagpie_vllm_omni/model_repository/easymp/config.pbtxt index f076bd9a481b..fbf8ecc897a1 100644 --- a/examples/tts/easymagpie_vllm_omni/model_repository/easymp/config.pbtxt +++ b/examples/tts/easymagpie_vllm_omni/model_repository/easymp/config.pbtxt @@ -134,3 +134,12 @@ parameters { key: "lt_top_k" value: { string_value: "80" } } + +# Benchmark-only toggle. When "true", the codec BLS is skipped entirely (no +# GPU->CPU copy, no codec inference); responses carry silence. Use this to +# measure the AR / orchestration path of model.py in isolation and compare it +# against benchmark_model.py. Leave "false" for real serving. +parameters { + key: "codec_noop" + value: { string_value: "false" } +} From 2429b049f9d6976c8bdf2c8646a3e620e532db40 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimkov Date: Mon, 15 Jun 2026 17:57:52 +0200 Subject: [PATCH 41/45] examples/tts/easymagpie_vllm_omni: extend benchmarking scripts Signed-off-by: Viacheslav Klimkov --- .../easymagpie_vllm_omni/benchmark_model.py | 202 +++++++++++------- .../easymagpie_vllm_omni/benchmark_service.py | 123 +++++++++-- 2 files changed, 233 insertions(+), 92 deletions(-) diff --git a/examples/tts/easymagpie_vllm_omni/benchmark_model.py b/examples/tts/easymagpie_vllm_omni/benchmark_model.py index 770c034bcee6..641192b52a56 100644 --- a/examples/tts/easymagpie_vllm_omni/benchmark_model.py +++ b/examples/tts/easymagpie_vllm_omni/benchmark_model.py @@ -62,6 +62,7 @@ DTYPE = "float16" STAGE_INIT_TIMEOUT = 300 # vLLM CUDA-graph capture strategy; None == vLLM default (FULL_AND_PIECEWISE). +#CUDAGRAPH_MODE: Optional[str] = "PIECEWISE" CUDAGRAPH_MODE: Optional[str] = None DEFAULT_PROMPTS = [ @@ -166,7 +167,8 @@ def _write_temp_stage_config(cfg: dict) -> str: @dataclass class ModelMeta: tokenizer: Any - speaker_embedding: Any # torch.Tensor (T_audio, embedding_dim) + speaker_embedding: Any # torch.Tensor (T_audio, embedding_dim); None in speaker_id mode + speaker_id: Optional[str] # known-speaker id (None => pass raw speaker_embedding) prompt_len: int audio_eos_id: int speech_delay: int @@ -175,7 +177,12 @@ class ModelMeta: text_eos_id: int # appended to streamed subword ids -def _load_model_meta(model_dir: str) -> ModelMeta: +def _load_model_meta( + model_dir: str, + lim_prefill: Optional[int] = None, + speaker_id: str = SPEAKER, + use_spkr_emb: bool = False, +) -> ModelMeta: import torch from transformers import AutoTokenizer @@ -186,23 +193,45 @@ def _load_model_meta(model_dir: str) -> ModelMeta: config = json.loads((model_path / "config.json").read_text()) arch = EasyMagpieOmniArch.from_hf_config(type("Cfg", (), config)) - emb_path = model_path / "speaker_embeddings" / f"{SPEAKER}.pt" - if not emb_path.exists(): - raise FileNotFoundError(f"Speaker embedding not found: {emb_path}") - loaded = torch.load(emb_path, map_location="cpu") - speaker_embedding = (loaded["speaker_encoding"] if isinstance(loaded, dict) else loaded).to(torch.float32) - + # Default: pass the known ``speaker_id`` (the model holds the embedding as + # precomputed state). Ship the raw tensor instead when explicitly requested + # (--use-spkr-emb) or when ``lim_prefill`` truncates it (so it no longer + # matches the registered embedding). + use_id = not (use_spkr_emb or lim_prefill is not None) tokenizer = AutoTokenizer.from_pretrained(model_dir, trust_remote_code=True) - prompt_len = EasyMagpieTTSForConditionalGeneration.estimate_prompt_len( - speaker_embedding, - tokenize=lambda t: tokenizer.encode(t), - context_text=CONTEXT_TEXT, - has_task_embedding=arch.num_task_embeddings > 0, - ) + + if use_id: + # Known-speaker flow: the caller never loads the embedding tensor — it + # passes the speaker_id and the model holds the embedding as state. + speaker_embedding = None + prompt_len = EasyMagpieTTSForConditionalGeneration.get_prompt_len( + speaker_id, + model_dir, + tokenize=lambda t: tokenizer.encode(t), + ) + else: + emb_path = model_path / "speaker_embeddings" / f"{speaker_id}.pt" + if not emb_path.exists(): + raise FileNotFoundError(f"Speaker embedding not found: {emb_path}") + loaded = torch.load(emb_path, map_location="cpu") + speaker_embedding = (loaded["speaker_encoding"] if isinstance(loaded, dict) else loaded).to(torch.float32) + # Optionally cap the speaker-embedding prefill to the first ``lim_prefill`` + # frames (mimics a single-token custom-voice prefill, cf. Qwen3-TTS). + if lim_prefill is not None: + orig_frames = int(speaker_embedding.shape[0]) + speaker_embedding = speaker_embedding[: max(1, int(lim_prefill))].contiguous() + logger.info("Limiting speaker-embedding prefill: %d -> %d frames", orig_frames, speaker_embedding.shape[0]) + prompt_len = EasyMagpieTTSForConditionalGeneration.estimate_prompt_len( + speaker_embedding, + tokenize=lambda t: tokenizer.encode(t), + context_text=CONTEXT_TEXT, + has_task_embedding=arch.num_task_embeddings > 0, + ) return ModelMeta( tokenizer=tokenizer, speaker_embedding=speaker_embedding, + speaker_id=speaker_id if use_id else None, prompt_len=int(prompt_len), audio_eos_id=int(arch.audio_eos_id), speech_delay=int(getattr(arch, "streaming_speech_delay", 0) or 0), @@ -213,16 +242,25 @@ def _load_model_meta(model_dir: str) -> ModelMeta: def build_prompt(text: str, meta: ModelMeta) -> dict: - return { - "prompt_token_ids": [0] * meta.prompt_len, - "additional_information": { - "speaker_embedding": meta.speaker_embedding, - "context_text": CONTEXT_TEXT, - "text": text, - "temperature": LT_TEMPERATURE, - "top_k": LT_TOPK, - }, + # Known-speaker path: pass ``speaker_id`` (the model holds the speaker's + # context-audio embedding as precomputed state) instead of shipping the + # ``(T_audio, embedding_dim)`` tensor per request. Falls back to a raw + # ``speaker_embedding`` tensor only when ``--use-spkr-emb`` is set. + info: dict = { + "context_text": CONTEXT_TEXT, + "text": text, + "temperature": LT_TEMPERATURE, + "top_k": LT_TOPK, } + info.update(_speaker_info(meta)) + return {"prompt_token_ids": [0] * meta.prompt_len, "additional_information": info} + + +def _speaker_info(meta: ModelMeta) -> dict: + """Speaker identifier passed in ``additional_information`` (id vs. raw tensor).""" + if meta.speaker_id is not None: + return {"speaker_id": meta.speaker_id} + return {"speaker_embedding": meta.speaker_embedding} # --------------------------------------------------------------------------- @@ -249,26 +287,14 @@ def _extract_request_output(stage_output): return getattr(stage_output, "request_output", stage_output) -def _new_audio_frames(payload, prev: int, cur: int): - """New decode frames since the previous step as a ``(k, C*S)`` tensor (or None). +class StepMeter: + """Cheap per-request measurement: TTFT, ITL, generated-frame count. - DELTA mode exposes ``audio_codes`` as a growing list ``[prefill, frame_0, ...]`` - during decode; prefill/final steps yield a cumulative tensor instead. + Deliberately does **no** per-step tensor work. End-of-generation (audio-EOS / + backbone stop token) is detected engine-side via ``stop_token_ids``; here we + only read timestamps and the running token count so the measurement loop never + blocks the engine and the throughput number reflects the model, not the harness. """ - import torch - - if cur - prev <= 0: - return None - if isinstance(payload, list): - chunks = [c for c in payload[1 + prev : 1 + cur] if isinstance(c, torch.Tensor) and c.numel() > 0] - return torch.cat(chunks, dim=0) if chunks else None - if isinstance(payload, torch.Tensor) and payload.shape[0] >= cur - prev: - return payload[-(cur - prev) :] - return None - - -class StepMeter: - """Shared per-request measurement: TTFT, ITL, audio length, audio-EOS.""" def __init__(self, meta: ModelMeta): self.meta = meta @@ -277,22 +303,21 @@ def __init__(self, meta: ModelMeta): self._t_start = time.perf_counter() self._t_last = None self._prev_tokens = 0 - self._eos_idx = None - - @property - def eos_reached(self) -> bool: - return self._eos_idx is not None + self._finish_reason = None def observe(self, stage_output) -> None: now = time.perf_counter() ro = _extract_request_output(stage_output) self.steps += 1 - cur = self._prev_tokens - if getattr(ro, "outputs", None): - out0 = ro.outputs[0] - cum = getattr(out0, "cumulative_token_ids", None) - cur = len(cum) if cum is not None else len(getattr(out0, "token_ids", []) or []) + out0 = ro.outputs[0] if getattr(ro, "outputs", None) else None + if out0 is None: + return + fr = getattr(out0, "finish_reason", None) + if fr is not None: + self._finish_reason = fr + cum = getattr(out0, "cumulative_token_ids", None) + cur = len(cum) if cum is not None else self._prev_tokens + len(getattr(out0, "token_ids", []) or []) if cur <= self._prev_tokens: return @@ -301,16 +326,6 @@ def observe(self, stage_output) -> None: else: self.result.inter_token_latencies.append(now - self._t_last) self._t_last = now - - mm = getattr(stage_output, "multimodal_output", None) or {} - new_frames = _new_audio_frames(mm.get("audio_codes"), self._prev_tokens, cur) - if new_frames is not None and new_frames.numel() > 0 and self._eos_idx is None: - for j in range(new_frames.shape[0]): - frame_idx = self._prev_tokens + j - if frame_idx >= self.meta.speech_delay and bool((new_frames[j] == self.meta.audio_eos_id).any()): - self._eos_idx = frame_idx - self.result.eos_reached = True - break self._prev_tokens = cur def finalize(self) -> RequestResult: @@ -318,8 +333,10 @@ def finalize(self) -> RequestResult: self.result.success = True if self.result.ttft_s == 0.0 and self.steps > 0: self.result.ttft_s = e2e_s - last_frame = self._eos_idx if self._eos_idx is not None else self._prev_tokens - audio_frames = max(0, last_frame - self.meta.speech_delay) + # "stop" => generation ended on the backbone stop token (audio-EOS); + # "length" => hit max_tokens without reaching EOS. + self.result.eos_reached = self._finish_reason == "stop" + audio_frames = max(0, self._prev_tokens - self.meta.speech_delay) self.result.audio_s = audio_frames * self.meta.frame_stacking_factor / CODEC_FRAME_RATE return self.result @@ -341,7 +358,10 @@ async def run_one_request( ``inputs`` is the prompt dict (whole-text) or an async generator of ``StreamingInput`` chunks; ``pace`` (streaming only) is awaited after each - frame to release the next chunk. Stops at audio-EOS / backbone stop token. + frame to release the next chunk. Termination is engine-driven: vLLM stops the + request at the backbone stop token (audio-EOS) or ``max_tokens``, which ends + the output stream — we just iterate to completion. ``max_steps`` is a streaming + safety valve only. """ meter = StepMeter(meta) gen = None @@ -349,10 +369,6 @@ async def run_one_request( gen = omni.generate(inputs, sampling_params_list=[sampling_params], request_id=request_id) async for stage_output in gen: meter.observe(stage_output) - ro = _extract_request_output(stage_output) - co = ro.outputs[0] if getattr(ro, "outputs", None) else None - if meter.eos_reached or getattr(co, "stop_reason", None) == meta.stop_token_id: - break if max_steps is not None and meter.steps >= max_steps: break if pace is not None: @@ -391,11 +407,11 @@ def build_streaming_request(text: str, meta: ModelMeta, stream_params, max_new_t from vllm.v1.engine.async_llm import StreamingInput prefill_info = { - "speaker_embedding": meta.speaker_embedding, "context_text": CONTEXT_TEXT, "temperature": LT_TEMPERATURE, "top_k": LT_TOPK, } + prefill_info.update(_speaker_info(meta)) text_ids = list(meta.tokenizer.encode(text, add_special_tokens=False)) + [meta.text_eos_id] tail_params = _clone_sampling_params(stream_params, max_new_tokens - len(text_ids)) go_queue: asyncio.Queue = asyncio.Queue() @@ -560,7 +576,13 @@ async def main(args): return logger.info("Loaded %d texts", len(texts)) - meta = _load_model_meta(args.model) + meta = _load_model_meta( + args.model, lim_prefill=args.lim_prefill, speaker_id=args.speaker_id, use_spkr_emb=args.use_spkr_emb + ) + logger.info( + "Speaker mode: %s", + f"known speaker_id={meta.speaker_id!r}" if meta.speaker_id else "raw speaker_embedding tensor per request", + ) logger.info( "prompt_len=%d audio_eos_id=%d speech_delay=%d frame_stacking=%d", meta.prompt_len, meta.audio_eos_id, meta.speech_delay, meta.frame_stacking_factor, @@ -580,12 +602,26 @@ async def main(args): ) tmp_config_path = _write_temp_stage_config(stage_cfg) + # With dummy (random) weights the backbone emits the audio-EOS stop token at + # random steps, so requests finish early at random lengths. To force every + # request to run the full, fixed number of decode steps we DROP the stop + # token (instead of pinning min_tokens): the model repurposes a 2-token dummy + # backbone vocab, and vLLM's min_tokens processor would -inf-mask the whole + # ``all_stop_token_ids`` set — which includes the tokenizer's real eos id + # (~151k) — indexing far outside the 2-wide logits tensor and tripping a CUDA + # device-side assert. With no stop token and ignore_eos, only max_tokens ends + # the request, giving exactly ``max_new_tokens`` steps. + is_dummy = args.load_format == "dummy" + stop_token_ids = [] if is_dummy else [meta.stop_token_id] + if is_dummy: + logger.info("Dummy weights: dropping stop token; every request runs exactly %d steps", args.max_new_tokens) + sampling_params = SamplingParams( temperature=0.0, max_tokens=args.max_new_tokens, detokenize=False, ignore_eos=True, - stop_token_ids=[meta.stop_token_id], + stop_token_ids=stop_token_ids, output_kind=RequestOutputKind.DELTA, ) # Streaming: max_tokens=1 -> one chunk per decoded frame. @@ -594,7 +630,7 @@ async def main(args): max_tokens=1, detokenize=False, ignore_eos=True, - stop_token_ids=[meta.stop_token_id], + stop_token_ids=stop_token_ids, output_kind=RequestOutputKind.DELTA, ) @@ -664,8 +700,28 @@ def parse_args(): parser.add_argument("--num-warmups", type=int, default=3, help="Warmup rounds (total = concurrency * this)") parser.add_argument("--no-warmup", action="store_true", help="Skip warmup") parser.add_argument("--max-new-tokens", type=int, default=256, help="Max decode frames per request") + parser.add_argument( + "--lim-prefill", + type=int, + default=None, + help="Cap the speaker-embedding prefill to the first N frames (default: no limit). " + "Use e.g. --lim-prefill 1 to mimic a single-token custom-voice prefill.", + ) + parser.add_argument( + "--speaker-id", + type=str, + default=SPEAKER, + help="Known speaker id (string) passed in the prompt; the model holds its embedding as " + "precomputed state (default: %(default)s).", + ) + parser.add_argument( + "--use-spkr-emb", + action="store_true", + help="Ship the raw speaker_embedding tensor per request instead of the known speaker_id " + "(exercises the custom-voice path).", + ) parser.add_argument("--max-model-len", type=int, default=1024) - parser.add_argument("--max-num-batched-tokens", type=int, default=1024) + parser.add_argument("--max-num-batched-tokens", type=int, default=4096) parser.add_argument( "--load-format", type=str, diff --git a/examples/tts/easymagpie_vllm_omni/benchmark_service.py b/examples/tts/easymagpie_vllm_omni/benchmark_service.py index f404ed9ecdd4..06fd15d0df87 100644 --- a/examples/tts/easymagpie_vllm_omni/benchmark_service.py +++ b/examples/tts/easymagpie_vllm_omni/benchmark_service.py @@ -46,6 +46,9 @@ class RequestResult: duration_s: float ttfa_s: float = 0.0 error: str | None = None + # Arrival time (relative to request start) and playback duration of each audio chunk. + chunk_arrivals: list[float] = field(default_factory=list) + chunk_durations: list[float] = field(default_factory=list) @dataclass @@ -80,6 +83,8 @@ def synthesize( t0 = time.perf_counter() t_first: float | None = None chunks: list[np.ndarray] = [] + chunk_arrivals: list[float] = [] + chunk_durations: list[float] = [] client.async_stream_infer( model_name=MODEL_NAME, @@ -92,17 +97,20 @@ def synthesize( result, error = result_q.get(timeout=chunk_timeout) except queue.Empty: elapsed = time.perf_counter() - t0 - return None, elapsed, elapsed, "no chunk within chunk_timeout" + return None, elapsed, elapsed, [], [], "no chunk within chunk_timeout" if error: elapsed = time.perf_counter() - t0 - return None, elapsed, elapsed, str(error) + return None, elapsed, elapsed, [], [], str(error) audio = result.as_numpy("audio").squeeze() if audio.size > 0: + now = time.perf_counter() if t_first is None: - t_first = time.perf_counter() + t_first = now chunks.append(audio) + chunk_arrivals.append(now - t0) + chunk_durations.append(audio.size / SAMPLE_RATE) response = result.get_response() final_param = response.parameters.get("triton_final_response") @@ -112,7 +120,7 @@ def synthesize( elapsed = time.perf_counter() - t0 ttfa = (t_first - t0) if t_first else elapsed audio = np.concatenate(chunks) if chunks else np.array([], dtype=np.float32) - return audio, ttfa, elapsed, None + return audio, ttfa, elapsed, chunk_arrivals, chunk_durations, None def worker( @@ -138,7 +146,9 @@ def worker( task_idx = task_queue.pop() uttid, text = random.choice(items) - audio, ttfa, elapsed, error = synthesize(client, result_q, text, chunk_timeout) + audio, ttfa, elapsed, chunk_arrivals, chunk_durations, error = synthesize( + client, result_q, text, chunk_timeout + ) if error is not None: client.stop_stream() @@ -152,7 +162,16 @@ def worker( if output_dir is not None and num_samples > 0: _save_wav(output_dir / f"{uttid}.wav", audio) - stats.add(RequestResult(uttid=uttid, num_samples=num_samples, duration_s=elapsed, ttfa_s=ttfa)) + stats.add( + RequestResult( + uttid=uttid, + num_samples=num_samples, + duration_s=elapsed, + ttfa_s=ttfa, + chunk_arrivals=chunk_arrivals, + chunk_durations=chunk_durations, + ) + ) if verbose: print( f"[worker {worker_id:02d}] req {task_idx} ({uttid}) — " @@ -214,31 +233,98 @@ def _percentile(sorted_vals: list[float], pct: float) -> float: return sorted_vals[idx] +def _mean(vals: list[float]) -> float: + return (sum(vals) / len(vals)) if vals else 0.0 + + +def _playback_metrics(arrivals: list[float], durations: list[float]) -> dict: + """Simulate gapless playback and detect buffer underruns. + + Playback starts when the first chunk arrives. An underrun occurs whenever a + chunk has not arrived yet by the time we finish playing everything buffered + so far (i.e. the player would stall waiting for it). Also reports the + inter-chunk gaps and the per-chunk realtime factor (playback time of a chunk + divided by the time it took to fetch it; >1 means we receive faster than we + play, so the stream is sustainable). + """ + n = len(arrivals) + if n == 0: + return {"chunks": 0, "underruns": 0, "gaps": [], "chunk_rtfs": []} + + underruns = 0 + gaps: list[float] = [] + chunk_rtfs: list[float] = [] + playback_end = arrivals[0] + durations[0] + for i in range(1, n): + gap = arrivals[i] - arrivals[i - 1] + gaps.append(gap) + if gap > 0: + chunk_rtfs.append(durations[i] / gap) + if arrivals[i] > playback_end: + underruns += 1 + playback_end = arrivals[i] + playback_end += durations[i] + return {"chunks": n, "underruns": underruns, "gaps": gaps, "chunk_rtfs": chunk_rtfs} + + def _summarize(stats: BenchmarkStats, wall_s: float, concurrency: int) -> dict: successes = [r for r in stats.results if r.error is None] failures = [r for r in stats.results if r.error is not None] audio_s = sum(r.num_samples for r in successes) / SAMPLE_RATE - ttfas_ms = sorted(r.ttfa_s * 1000 for r in successes) + ttfts_ms = sorted(r.ttfa_s * 1000 for r in successes) + + itl_ms: list[float] = [] + chunk_rtfs: list[float] = [] + total_chunks = 0 + total_underruns = 0 + reqs_with_underrun = 0 + for r in successes: + pm = _playback_metrics(r.chunk_arrivals, r.chunk_durations) + total_chunks += pm["chunks"] + total_underruns += pm["underruns"] + if pm["underruns"] > 0: + reqs_with_underrun += 1 + itl_ms.extend(g * 1000 for g in pm["gaps"]) + chunk_rtfs.extend(pm["chunk_rtfs"]) + itl_ms.sort() + return { "concurrency": concurrency, + "ok": len(successes), "failed": len(failures), "wall_s": wall_s, "audio_s": audio_s, - "rtx": audio_s / wall_s if wall_s > 0 else 0.0, + "rtf": audio_s / wall_s if wall_s > 0 else 0.0, "tput": len(successes) / wall_s if wall_s > 0 else 0.0, - "ttfa_mean_ms": (sum(ttfas_ms) / len(ttfas_ms)) if ttfas_ms else 0.0, - "ttfa_p95_ms": _percentile(ttfas_ms, 0.95), + "ttft_mean_ms": _mean(ttfts_ms), + "ttft_p95_ms": _percentile(ttfts_ms, 0.95), + "itl_mean_ms": _mean(itl_ms), + "itl_p95_ms": _percentile(itl_ms, 0.95), + "total_chunks": total_chunks, + "total_underruns": total_underruns, + "reqs_with_underrun": reqs_with_underrun, + "underrun_pct": (100.0 * total_underruns / total_chunks) if total_chunks else 0.0, + "playback_rtf_mean": _mean(chunk_rtfs), } def _print_summary(s: dict): print( - f"[concurrency={s['concurrency']}] rtx = synt / time = " - f"{s['rtx']:.2f}x = {s['audio_s']:.0f} / {s['wall_s']:.2f}" + f"concurrency={s['concurrency']}: req/s {s['tput']:.2f}, " + f"ttft {s['ttft_mean_ms']:.1f}ms, itl {s['itl_mean_ms']:.1f}ms, " + f"rtf {s['rtf']:.2f}x, underrun {s['underrun_pct']:.1f}%" ) + + +def _print_detailed(s: dict): + print(f"[concurrency={s['concurrency']}] {s['ok']} ok / {s['failed']} failed") + print(f" req/s {s['tput']:.2f} | rtf {s['rtf']:.2f}x (audio {s['audio_s']:.0f}s / wall {s['wall_s']:.2f}s)") + print(f" ttft mean {s['ttft_mean_ms']:.1f}ms p95 {s['ttft_p95_ms']:.1f}ms") + print(f" itl mean {s['itl_mean_ms']:.1f}ms p95 {s['itl_p95_ms']:.1f}ms") print( - f"throughput={s['tput']:.2f} req/s; failed = {s['failed']}; " - f"TTFA={s['ttfa_mean_ms']:.1f} / {s['ttfa_p95_ms']:.1f} (p95)" + f" playback underruns {s['total_underruns']}/{s['total_chunks']} chunks " + f"({s['underrun_pct']:.2f}%) in {s['reqs_with_underrun']}/{s['ok']} reqs | " + f"realtime factor mean {s['playback_rtf_mean']:.2f}x (chunk play / fetch)" ) @@ -278,12 +364,11 @@ def main(): ) summary = _summarize(stats, wall_elapsed, concurrency) summaries.append(summary) - _print_summary(summary) + _print_detailed(summary) - if len(summaries) > 1: - print("\n=== Summary ===") - for s in summaries: - _print_summary(s) + print("\n=== Summary ===") + for s in summaries: + _print_summary(s) if __name__ == "__main__": From 332579a9ec36707ceb3debad4def26d52a9c10cd Mon Sep 17 00:00:00 2001 From: Viacheslav Klimkov Date: Tue, 16 Jun 2026 00:50:54 +0200 Subject: [PATCH 42/45] easymagpie_vllm_omni/easymagpie.py: simplify preprocessing, expect list of text tokens Signed-off-by: Viacheslav Klimkov --- .../easymagpie_vllm_omni/easymagpie.py | 111 +++++++----------- 1 file changed, 40 insertions(+), 71 deletions(-) diff --git a/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/easymagpie.py b/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/easymagpie.py index a7f9977059fd..6e97d3e49a57 100644 --- a/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/easymagpie.py +++ b/examples/tts/easymagpie_vllm_omni/easymagpie_vllm_omni/easymagpie.py @@ -613,43 +613,6 @@ def make_omni_output(self, model_outputs, **_: Any) -> OmniOutput: # preprocess / postprocess # ------------------------------------------------------------------ - @staticmethod - def _first_str(value: Any) -> str: - """Return the first element of a list-wrapped scalar, or the scalar itself, as a string.""" - if isinstance(value, list): - return str(value[0]) if value else "" - if value is None: - return "" - return str(value) - - @staticmethod - def _coerce_opt_int(value: Any) -> Optional[int]: - """Best-effort extract a single int from a scalar / list / tensor / str. - - Used to read a per-step streamed ``text_token`` out of the request's - ``additional_information`` (which may wrap the id as a list, a 1-element - tensor, or a string depending on how the caller / transport packed it). - Returns ``None`` when no usable integer is present. - """ - if value is None: - return None - if isinstance(value, bool): # bool is an int subclass — handle explicitly. - return int(value) - if isinstance(value, int): - return value - if isinstance(value, float): - return int(value) - if isinstance(value, torch.Tensor): - return int(value.reshape(-1)[0].item()) if value.numel() > 0 else None - if isinstance(value, (list, tuple)): - return EasyMagpieTTSForConditionalGeneration._coerce_opt_int(value[0]) if value else None - if isinstance(value, str): - try: - return int(value.strip()) - except ValueError: - return None - return None - def preprocess( self, input_ids: torch.Tensor, @@ -744,7 +707,7 @@ def _preprocess_prefill( # no list is baked, and :meth:`_preprocess_decode` instead reads one # subword id per step from the streamed ``additional_information.text_token``. if not info_dict.get("text_tokens"): - text = self._first_str(info_dict.get("text")) + text = info_dict.get("text") if text: info_update["text_tokens"] = self._encode_text_stream(text) input_ids_out = torch.full_like(input_ids, _DUMMY_TOKEN_ID) @@ -787,7 +750,7 @@ def _build_prefill_embeds( parts.append(self._resolve_speaker_embedding(device, info_dict)) # Context text: tokenized in-model and embedded through the baked table. - context_text = self._first_str(info_dict.get("context_text")) or _DEFAULT_CONTEXT_TEXT + context_text = info_dict.get("context_text") or _DEFAULT_CONTEXT_TEXT ctx_ids = self._encode_context_text(context_text, device) if ctx_ids.numel() > 0: parts.append(self.text_embedding(ctx_ids).to(dtype)) @@ -803,7 +766,7 @@ def _resolve_speaker_embedding(self, device: torch.device, info_dict: dict[str, here. Exactly one of the two must be supplied. """ dtype = self._combined_embeddings.dtype - speaker_id = self._first_str(info_dict.get("speaker_id")) + speaker_id = info_dict.get("speaker_id") if speaker_id: embedding = self._speaker_embeddings.get(speaker_id) assert embedding is not None, ( @@ -866,10 +829,10 @@ def _maybe_set_lt_sampling_params(self, info_dict: dict[str, Any]) -> None: """ temperature = info_dict.get("temperature") if temperature is not None: - self.code_predictor.temperature = float(self._first_str(temperature) or 0.0) + self.code_predictor.temperature = float(temperature) top_k = info_dict.get("top_k", info_dict.get("topk")) if top_k is not None: - self.code_predictor.top_k = int(float(self._first_str(top_k) or 0)) + self.code_predictor.top_k = int(top_k) def _get_text_tokenizer(self): """Lazily load the context-text tokenizer from the model directory. @@ -989,39 +952,45 @@ def _preprocess_decode( info_update: dict[str, Any] = {"decode_offset": decode_offset + 1} # ── Text channel ── (delay 0: one subword per step from step 0). The text - # stream leads the phoneme/audio streams by their respective delays. Two - # mutually exclusive input modes are supported: + # stream leads the phoneme/audio streams by their respective delays. The + # model always consumes exactly one buffered subword id per decode step, + # indexed by ``decode_offset`` from a persistent ``text_tokens`` list. That + # list is populated by one of two mutually exclusive input modes: # # * **Whole-text (non-streaming)** — the caller passed ``text`` whole at # prefill; it was tokenized in-model and stashed as the ``text_tokens`` - # list (see :meth:`_preprocess_prefill`). Step k consumes - # ``text_tokens[k]`` (the list ends with the text-EOS id); once the - # stream is exhausted the channel is masked off (adds nothing) rather - # than repeating the last token. - # * **Streamed** — the caller did *not* pass ``text`` at prefill and - # instead pushes one subword id per decode step via - # ``additional_information`` under ``text_token`` (a single int / 1-elem - # tensor; close the stream by pushing the text-EOS id as the last real - # token). The model embeds that step's id and masks the channel off on - # any step that carries no id (``text_token`` absent or ``< 0``), so the - # caller can keep pumping decode steps after the text ends while the - # audio tail finishes. Because each streamed chunk overwrites the - # previous ``text_token`` in the per-request buffer, every step gets a - # fresh value (or the caller's sentinel ``-1`` to mask). - text_tokens = info_dict.get("text_tokens") - if isinstance(text_tokens, list): - if decode_offset < len(text_tokens): - self._dec_text_tokens[start] = int(text_tokens[decode_offset]) - self._dec_text_mask[start] = 1 - else: - self._dec_text_mask[start] = 0 + # list (see :meth:`_preprocess_prefill`). No per-step ``text_token`` + # arrives, so the buffer never grows here. + # * **Streamed** — the caller did *not* pass ``text`` at prefill and instead + # pushes subword ids during decode via ``additional_information`` under + # ``text_token`` (always a ``list[int]``). Each chunk may carry a single id + # (``[id]`` with ``max_tokens == 1``, one frame per chunk) or several ids at + # once (``max_tokens == N``, so the engine free-runs N frames off one chunk — + # fewer round-trips). Those ids are appended to ``text_tokens`` and consumed + # one per step. + # + # In both modes, once the buffer is exhausted the channel is masked off + # (adds nothing) rather than repeating the last token, so the caller can keep + # pumping decode steps (passing an empty ``text_token`` list) while the audio + # tail finishes. + # + # Append-once: a streamed chunk's ``text_token`` payload stays identical on + # every decode step of that chunk, so we extend the buffer only when it has + # been fully consumed (``decode_offset`` caught up to its length). This is + # safe because the chunk's segment stops at ``max_tokens`` and ``preprocess`` + # is not called again until a fresh chunk has replaced ``text_token`` — hence + # the caller must size each chunk's ``max_tokens`` to the number of ids it + # carries. + text_tokens = info_dict.get("text_tokens") or [] + incoming = info_dict.get("text_token") or [] + if incoming and decode_offset >= len(text_tokens): + text_tokens = text_tokens + [int(t) for t in incoming] + info_update["text_tokens"] = text_tokens + if decode_offset < len(text_tokens): + self._dec_text_tokens[start] = int(text_tokens[decode_offset]) + self._dec_text_mask[start] = 1 else: - streamed_id = self._coerce_opt_int(info_dict.get("text_token")) - if streamed_id is not None and streamed_id >= 0: - self._dec_text_tokens[start] = streamed_id - self._dec_text_mask[start] = 1 - else: - self._dec_text_mask[start] = 0 + self._dec_text_mask[start] = 0 # ── Phoneme channel ── opens at decode step == ``phonemes_delay`` (seeded # with phoneme BOS), then feeds back the previous step's prediction, and From 39a40058aac4ca6740b1d4a015cee359c691133f Mon Sep 17 00:00:00 2001 From: Viacheslav Klimkov Date: Tue, 16 Jun 2026 00:51:41 +0200 Subject: [PATCH 43/45] examples/tts/easymagpie_vllm_omni: allow to feed chunks of text tokens Signed-off-by: Viacheslav Klimkov --- .../easymagpie_inference_demo.ipynb | 476 +++++++++++++++--- .../model_repository/easymp/1/model.py | 109 ++-- 2 files changed, 485 insertions(+), 100 deletions(-) diff --git a/examples/tts/easymagpie_vllm_omni/easymagpie_inference_demo.ipynb b/examples/tts/easymagpie_vllm_omni/easymagpie_inference_demo.ipynb index 6f4b746aa655..9fe6d5b13bc3 100644 --- a/examples/tts/easymagpie_vllm_omni/easymagpie_inference_demo.ipynb +++ b/examples/tts/easymagpie_vllm_omni/easymagpie_inference_demo.ipynb @@ -18,10 +18,27 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "id": "c9a71b74", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/vklimkov/miniconda3/envs/emp/lib/python3.12/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n", + "Unable to import `torchao` Tensor objects. This may affect loading checkpoints serialized with `torchao`\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "torch: 2.11.0+cu130 | cuda: True\n" + ] + } + ], "source": [ "import os\n", "\n", @@ -58,10 +75,23 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "id": "3e0df89e", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Model dir : /home/vklimkov/workspace/emp/NeMo/examples/tts/easymagpie_vllm_omni/easymp_vllm_model\n", + "num_stacked_codebooks : 16 (C*S)\n", + "audio_bos / audio_eos id : 1024 / 1025\n", + "text_vocab / text_eos : 131075 / 131073\n", + "audio-EOS stop token id : 1\n", + "streaming speech delay : 5 frames\n" + ] + } + ], "source": [ "MODEL_DIR = Path(\"/home/vklimkov/workspace/emp/NeMo/examples/tts/easymagpie_vllm_omni/easymp_vllm_model\")\n", "assert (MODEL_DIR / \"config.json\").exists(), f\"No config.json under {MODEL_DIR}; run the converter first.\"\n", @@ -98,10 +128,168 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "id": "5085e9a4", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Stage config: /tmp/easymagpie_omni_demo_6qki61d4.yaml\n", + "INFO 06-15 18:51:09 [omni_base.py:172] [AsyncOmni] Initializing with model /home/vklimkov/workspace/emp/NeMo/examples/tts/easymagpie_vllm_omni/easymp_vllm_model\n", + "INFO 06-15 18:51:09 [async_omni_engine.py:269] [AsyncOmniEngine] Initializing with model /home/vklimkov/workspace/emp/NeMo/examples/tts/easymagpie_vllm_omni/easymp_vllm_model\n", + "WARNING 06-15 18:51:09 [utils.py:597] --stage-configs-path is deprecated; migrate '/tmp/easymagpie_omni_demo_6qki61d4.yaml' and use --deploy-config.\n", + "INFO 06-15 18:51:09 [async_omni_engine.py:331] [AsyncOmniEngine] Launching Orchestrator thread with 1 stages\n", + "INFO 06-15 18:51:09 [initialization.py:352] Loaded OmniTransferConfig with 0 connector configurations\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "OneLogger: Setting error_handling_strategy to DISABLE_QUIETLY_AND_REPORT_METRIC_ERROR for rank (rank=0) with OneLogger disabled. To override: explicitly set error_handling_strategy parameter.\n", + "No exporters were provided. This means that no telemetry data will be collected.\n", + "[transformers] Model config: eos_token_id must be `None` or an integer within the vocabulary (between 0 and 1), got 2. This may result in unexpected behavior.\n", + "[transformers] Model config: forced_mask_token_id must be `None` or an integer within the vocabulary (between 0 and 1), got 1028. This may result in unexpected behavior.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "INFO 06-15 18:51:18 [model.py:568] Resolved architecture: EasyMagpieTTSForConditionalGeneration\n", + "INFO 06-15 18:51:18 [model.py:2032] Downcasting torch.float32 to torch.float16.\n", + "INFO 06-15 18:51:18 [model.py:1697] Using max model len 1024\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2026-06-15 18:51:19,006\tINFO util.py:154 -- Missing packages: ['ipywidgets']. Run `pip install -U ipywidgets`, then restart the notebook server for rich notebook output.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "INFO 06-15 18:51:19 [scheduler.py:239] Chunked prefill is enabled with max_num_batched_tokens=1024.\n", + "INFO 06-15 18:51:19 [vllm.py:886] Asynchronous scheduling is enabled.\n", + "INFO 06-15 18:51:19 [kernel.py:212] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])\n", + "INFO 06-15 18:51:19 [stage_init_utils.py:535] [stage_init] Stage-0 set runtime devices: 0\n", + "INFO 06-15 18:51:19 [async_omni_engine.py:706] [AsyncOmniEngine] Stage 0 engine launch started\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "(StageEngineCoreProc pid=2884868) OneLogger: Setting error_handling_strategy to DISABLE_QUIETLY_AND_REPORT_METRIC_ERROR for rank (rank=0) with OneLogger disabled. To override: explicitly set error_handling_strategy parameter.\n", + "(StageEngineCoreProc pid=2884868) No exporters were provided. This means that no telemetry data will be collected.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "(StageEngineCoreProc pid=2884868) INFO 06-15 18:51:25 [core.py:109] Initializing a V1 LLM engine (v0.21.0) with config: model='/home/vklimkov/workspace/emp/NeMo/examples/tts/easymagpie_vllm_omni/easymp_vllm_model', speculative_config=None, tokenizer='/home/vklimkov/workspace/emp/NeMo/examples/tts/easymagpie_vllm_omni/easymp_vllm_model', skip_tokenizer_init=True, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.float16, max_seq_len=1024, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, quantization_config=None, enforce_eager=False, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False), seed=0, served_model_name=/home/vklimkov/workspace/emp/NeMo/examples/tts/easymagpie_vllm_omni/easymp_vllm_model, enable_prefix_caching=False, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'ir_enable_torch_wrap': True, 'splitting_ops': ['vllm::unified_attention_with_output', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::gdn_attention_core_xpu', 'vllm::olmo_hybrid_gdn_full_forward', 'vllm::kda_attention', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::deepseek_v4_attention', 'vllm::unified_kv_cache_update', 'vllm::unified_mla_kv_cache_update'], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [1024], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'size_asserts': False, 'alignment_asserts': False, 'scalar_asserts': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'fuse_act_padding': False}, 'max_cudagraph_capture_size': 2, 'dynamic_shapes_config': {'type': , 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']), enable_flashinfer_autotune=False, moe_backend='auto')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "(StageEngineCoreProc pid=2884868) Unable to import `torchao` Tensor objects. This may affect loading checkpoints serialized with `torchao`\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "(StageEngineCoreProc pid=2884868) INFO 06-15 18:51:25 [parallel_state.py:1410] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://10.221.143.184:50843 backend=nccl\n", + "(StageEngineCoreProc pid=2884868) INFO 06-15 18:51:25 [parallel_state.py:1723] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0, EPLB rank N/A\n", + "(StageEngineCoreProc pid=2884868) INFO 06-15 18:51:26 [topk_topp_sampler.py:45] Using FlashInfer for top-p & top-k sampling.\n", + "(StageEngineCoreProc pid=2884868) WARNING 06-15 18:51:26 [base.py:188] [LLM Worker 0] Sleep Mode DISABLED.\n", + "(StageEngineCoreProc pid=2884868) WARNING 06-15 18:51:26 [base.py:188] [LLM Worker 0] Sleep Mode DISABLED.\n", + "(StageEngineCoreProc pid=2884868) INFO 06-15 18:51:26 [gpu_model_runner.py:4857] Starting to load model /home/vklimkov/workspace/emp/NeMo/examples/tts/easymagpie_vllm_omni/easymp_vllm_model...\n", + "(StageEngineCoreProc pid=2884868) INFO 06-15 18:51:26 [unquantized.py:288] Using TRITON Unquantized MoE backend out of potential backends: ['FlashInfer TRTLLM', 'FlashInfer CUTLASS', 'TRITON', 'BATCHED_TRITON'].\n", + "(StageEngineCoreProc pid=2884868) INFO 06-15 18:51:26 [cuda.py:312] Using AttentionBackendEnum.TRITON_ATTN backend.\n", + "(StageEngineCoreProc pid=2884868) INFO 06-15 18:51:26 [weight_utils.py:938] Filesystem type for checkpoints: EXT4. Checkpoint size: 5.17 GiB. Available RAM: 100.95 GiB.\n", + "(StageEngineCoreProc pid=2884868) INFO 06-15 18:51:26 [weight_utils.py:961] Auto-prefetch is disabled because the filesystem (EXT4) is not a recognized network FS (NFS/Lustre). If you want to force prefetching, start vLLM with --safetensors-load-strategy=prefetch.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Loading safetensors checkpoint shards: 0% Completed | 0/1 [00:00= mamba page size.\n", + "(StageEngineCoreProc pid=2884868) INFO 06-15 18:51:32 [interface.py:669] Padding mamba page size by 1.39% to ensure that mamba page size and attention page size are exactly equal.\n", + "(StageEngineCoreProc pid=2884868) INFO 06-15 18:51:33 [backends.py:1089] Using cache directory: /home/vklimkov/.cache/vllm/torch_compile_cache/44b29fc96b/rank_0_0/backbone for vLLM's torch.compile\n", + "(StageEngineCoreProc pid=2884868) INFO 06-15 18:51:33 [backends.py:1148] Dynamo bytecode transform time: 0.33 s\n", + "(StageEngineCoreProc pid=2884868) INFO 06-15 18:51:33 [backends.py:292] Directly load the compiled graph(s) for compile range (1, 1024) from the cache, took 0.189 s\n", + "(StageEngineCoreProc pid=2884868) INFO 06-15 18:51:33 [decorators.py:311] Directly load AOT compilation from path /home/vklimkov/.cache/vllm/torch_compile_cache/torch_aot_compile/02d812b156948a92b817daa722171ad1f59c12f742f2ce3ace4f4e6d27b8d15c/rank_0_0/model\n", + "(StageEngineCoreProc pid=2884868) INFO 06-15 18:51:33 [monitor.py:53] torch.compile took 0.55 s in total\n", + "(StageEngineCoreProc pid=2884868) WARNING 06-15 18:51:33 [fused_moe.py:1073] Using default MoE config. Performance might be sub-optimal! Config file not found at /home/vklimkov/miniconda3/envs/emp/lib/python3.12/site-packages/vllm/model_executor/layers/fused_moe/configs/E=24,N=768,device_name=NVIDIA_RTX_A6000.json\n", + "(StageEngineCoreProc pid=2884868) INFO 06-15 18:51:34 [monitor.py:81] Initial profiling/warmup run took 0.71 s\n", + "(StageEngineCoreProc pid=2884868) INFO 06-15 18:51:34 [backends.py:1089] Using cache directory: /home/vklimkov/.cache/vllm/torch_compile_cache/44b29fc96b/rank_0_0/local_transformer for vLLM's torch.compile\n", + "(StageEngineCoreProc pid=2884868) INFO 06-15 18:51:34 [backends.py:1148] Dynamo bytecode transform time: 0.45 s\n", + "(StageEngineCoreProc pid=2884868) INFO 06-15 18:51:35 [backends.py:292] Directly load the compiled graph(s) for compile range (1, 1024) from the cache, took 0.362 s\n", + "(StageEngineCoreProc pid=2884868) INFO 06-15 18:51:35 [decorators.py:311] Directly load AOT compilation from path /home/vklimkov/.cache/vllm/torch_compile_cache/torch_aot_compile/f5fe91f41c1f0ea35f2560edb8e977d5715632d0296b883f1ce020652b24537e/rank_0_0/model\n", + "(StageEngineCoreProc pid=2884868) INFO 06-15 18:51:35 [monitor.py:53] torch.compile took 0.85 s in total\n", + "(StageEngineCoreProc pid=2884868) INFO 06-15 18:51:36 [monitor.py:81] Initial profiling/warmup run took 1.42 s\n", + "(StageEngineCoreProc pid=2884868) INFO 06-15 18:51:37 [base.py:142] Available KV cache memory: 34.89 GiB (process-scoped)\n", + "(StageEngineCoreProc pid=2884868) WARNING 06-15 18:51:37 [kv_cache_utils.py:1152] Add 1 padding layers, may waste at most 7.14% KV cache memory\n", + "(StageEngineCoreProc pid=2884868) INFO 06-15 18:51:37 [kv_cache_utils.py:1710] GPU KV cache size: 1,951,360 tokens\n", + "(StageEngineCoreProc pid=2884868) INFO 06-15 18:51:37 [kv_cache_utils.py:1711] Maximum concurrency for 1,024 tokens per request: 1905.62x\n", + "(StageEngineCoreProc pid=2884868) INFO 06-15 18:51:37 [ssu_dispatch.py:224] Using triton Mamba SSU backend.\n", + "(StageEngineCoreProc pid=2884868) INFO 06-15 18:51:37 [kernel_warmup.py:44] Skipping FlashInfer autotune because it is disabled.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 2/2 [00:00<00:00, 7.03it/s]\n", + "Capturing CUDA graphs (decode, FULL): 100%|██████████| 1/1 [00:00<00:00, 18.54it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "(StageEngineCoreProc pid=2884868) INFO 06-15 18:51:38 [gpu_model_runner.py:6243] Graph capturing finished in 1 secs, took 0.07 GiB\n", + "(StageEngineCoreProc pid=2884868) INFO 06-15 18:51:38 [jit_monitor.py:54] Kernel JIT monitor activated — Triton JIT compilations during inference will be logged as warnings.\n", + "(StageEngineCoreProc pid=2884868) INFO 06-15 18:51:38 [core.py:299] init engine (profile, create kv cache, warmup model) took 5.53 s (compilation: 1.41 s)\n", + "(StageEngineCoreProc pid=2884868) WARNING 06-15 18:51:38 [scheduler.py:181] Using custom scheduler class easymagpie_vllm_omni.scheduler.EasyMagpieARAsyncScheduler. This scheduler interface is not public and compatibility may not be maintained.\n", + "(StageEngineCoreProc pid=2884868) INFO 06-15 18:51:38 [factory.py:46] Created connector: SharedMemoryConnector\n", + "(StageEngineCoreProc pid=2884868) INFO 06-15 18:51:38 [vllm.py:886] Asynchronous scheduling is enabled.\n", + "(StageEngineCoreProc pid=2884868) INFO 06-15 18:51:38 [kernel.py:212] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])\n", + "INFO 06-15 18:51:38 [async_omni_engine.py:722] [AsyncOmniEngine] Stage 0 engine startup completed\n", + "INFO 06-15 18:51:38 [stage_engine_core_client.py:163] [StageEngineCoreClient] stage-0 [rep-0] initializing EngineCore\n", + "INFO 06-15 18:51:38 [stage_engine_core_client.py:206] [StageEngineCoreClient] stage-0 [rep-0] EngineCore running\n", + "INFO 06-15 18:51:38 [async_omni_engine.py:743] [AsyncOmniEngine] Stage 0 initialized\n", + "INFO 06-15 18:51:38 [orchestrator.py:164] [Orchestrator] Starting event loop\n", + "INFO 06-15 18:51:38 [async_omni_engine.py:358] [AsyncOmniEngine] Orchestrator ready with 1 stages\n", + "INFO 06-15 18:51:38 [omni_base.py:185] [AsyncOmni] AsyncOmniEngine initialized in 29.55 seconds\n", + "INFO 06-15 18:51:38 [omni_base.py:202] [AsyncOmni] Initialized with 1 stages for model /home/vklimkov/workspace/emp/NeMo/examples/tts/easymagpie_vllm_omni/easymp_vllm_model\n", + "Engine ready (single stage: EasyMagpie talker)\n" + ] + } + ], "source": [ "DECODE_STEPS = 256 # max audio frames to decode (trimmed at audio EOS)\n", "MAX_MODEL_LEN = 1024\n", @@ -181,10 +369,27 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "id": "697c74b3", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[transformers] The tokenizer you are loading from '/home/vklimkov/workspace/emp/NeMo/examples/tts/easymagpie_vllm_omni/easymp_vllm_model' with an incorrect regex pattern: https://huggingface.co/mistralai/Mistral-Small-3.1-24B-Instruct-2503/discussions/84#69121093e8b480e709447d5e. This will lead to incorrect tokenization. You should set the `fix_mistral_regex=True` flag when loading this tokenizer to fix this issue.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "speaker_embedding : (64, 1536)\n", + "context_text / text : '[EN]' / 'Hello, this is a test of the EasyMagpie text to speech model.'\n", + "prompt_len (placeholders) : 67\n" + ] + } + ], "source": [ "torch.manual_seed(0)\n", "\n", @@ -258,10 +463,61 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "id": "6d0ccbd4", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "WARNING 06-15 18:51:39 [input_processor.py:274] Passing raw prompts to InputProcessor is deprecated and will be removed in v0.18. You should instead pass the outputs of Renderer.render_cmpl() or Renderer.render_chat().\n", + "WARNING 06-15 18:51:39 [base.py:301] Using None for EOS token id because tokenizer is not initialized\n", + "INFO 06-15 18:51:39 [stage_engine_core_client.py:265] [StageEngineCoreClient] stage-0 [rep-0] add request: easymagpie-2fc385b4\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "(StageEngineCoreProc pid=2884868) [transformers] Model config: eos_token_id must be `None` or an integer within the vocabulary (between 0 and 1), got 2. This may result in unexpected behavior.\n", + "(StageEngineCoreProc pid=2884868) [transformers] Model config: forced_mask_token_id must be `None` or an integer within the vocabulary (between 0 and 1), got 1028. This may result in unexpected behavior.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "(StageEngineCoreProc pid=2884868) WARNING 06-15 18:51:39 [jit_monitor.py:103] Triton kernel JIT compilation during inference: _zero_kv_blocks_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.\n", + "(StageEngineCoreProc pid=2884868) WARNING 06-15 18:51:39 [gpu_model_runner.py:408] additional_information on request data is deprecated, use model_intermediate_buffer\n", + "(StageEngineCoreProc pid=2884868) WARNING 06-15 18:51:39 [jit_monitor.py:103] Triton kernel JIT compilation during inference: _compute_slot_mapping_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "(StageEngineCoreProc pid=2884868) [transformers] The tokenizer you are loading from '/home/vklimkov/workspace/emp/NeMo/examples/tts/easymagpie_vllm_omni/easymp_vllm_model' with an incorrect regex pattern: https://huggingface.co/mistralai/Mistral-Small-3.1-24B-Instruct-2503/discussions/84#69121093e8b480e709447d5e. This will lead to incorrect tokenization. You should set the `fix_mistral_regex=True` flag when loading this tokenizer to fix this issue.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "(StageEngineCoreProc pid=2884868) WARNING 06-15 18:51:40 [gpu_model_runner.py:1595] _merge_additional_information_update is deprecated, use _update_intermediate_buffer\n", + "(StageEngineCoreProc pid=2884868) WARNING 06-15 18:51:40 [jit_monitor.py:103] Triton kernel JIT compilation during inference: _causal_conv1d_fwd_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.\n", + "(StageEngineCoreProc pid=2884868) WARNING 06-15 18:51:40 [jit_monitor.py:103] Triton kernel JIT compilation during inference: _chunk_cumsum_fwd_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.\n", + "(StageEngineCoreProc pid=2884868) WARNING 06-15 18:51:40 [jit_monitor.py:103] Triton kernel JIT compilation during inference: _chunk_state_fwd_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.\n", + "(StageEngineCoreProc pid=2884868) WARNING 06-15 18:51:40 [jit_monitor.py:103] Triton kernel JIT compilation during inference: _state_passing_fwd_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.\n", + "(StageEngineCoreProc pid=2884868) WARNING 06-15 18:51:40 [jit_monitor.py:103] Triton kernel JIT compilation during inference: _bmm_chunk_fwd_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.\n", + "(StageEngineCoreProc pid=2884868) WARNING 06-15 18:51:40 [jit_monitor.py:103] Triton kernel JIT compilation during inference: _chunk_scan_fwd_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.\n", + "(StageEngineCoreProc pid=2884868) WARNING 06-15 18:51:40 [jit_monitor.py:103] Triton kernel JIT compilation during inference: fused_moe_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.\n", + "(StageEngineCoreProc pid=2884868) WARNING 06-15 18:51:40 [jit_monitor.py:103] Triton kernel JIT compilation during inference: kernel_unified_attention. This causes a latency spike; consider extending warmup to cover this shape/config.\n", + "cumulative codes : (132, 16) (prompt_len=67)\n", + "audio_codes : (59, 16) (dropped prefix + 5 warm-up + 1 EOS)\n" + ] + } + ], "source": [ "async def run_request(prompt, sampling_params):\n", " \"\"\"Keep the cumulative audio-code tensor [prefill_prefix | decoded frames].\n", @@ -290,10 +546,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "id": "04196662", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAw4AAAE8CAYAAABpb+bCAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjksIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvJkbTWQAAAAlwSFlzAAAPYQAAD2EBqD+naQAAbclJREFUeJzt3XlcVNX7B/DPsCM7KiAKiIoL7mkibmmSpLhbqVmuaQtoiqn5K/cFl9xzSSu10tQytdWNTDJ3iDQ1V1RcEDdAULaZ8/vDmK8TyDyjIAif9/d1X9+895nnnrl3Zphn7j3naJRSCkRERERERPkwK+oGEBERERFR8cfCgYiIiIiIjGLhQERERERERrFwICIiIiIio1g4EBERERGRUSwciIiIiIjIKBYORERERERkFAsHIiIiIiIyioUDEREREREZxcKBqBirXLkyOnbsWGD5zp8/D41Gg1WrVhVYTgJat26N1q1b6//9tB5njUaDiRMnFnUzAACpqalwc3PDmjVrimT/p0+fRrt27eDk5ASNRoPNmzcXSTsK0/Hjx2FhYYG///67qJtCRE8JFg5E9Fj27t2LiRMnIikpqVD3c/fuXUycOBG//fZboe6HiocFCxbAwcEBvXr10q9btWoVNBpNnktCQoLB41NTUzF8+HBUqlQJ1tbWqFWrFpYuXSref79+/XD06FFMmzYNX375JRo3blxgz6248Pf3R0hICMaPH1/UTSGip4RFUTeAiJ5ue/fuxaRJk9C/f384OzsX2n7u3r2LSZMmAYDBr/vFkY+PD+7duwdLS8uibspTKSsrCwsWLMCIESNgbm6ea/vkyZPh6+trsO7B155Wq0VwcDAOHz6M0NBQ+Pn5Ydu2bXjnnXdw+/Zt/N///V+++7937x727duHDz74AGFhYQXynIqrt956Cx06dMDZs2dRtWrVom4OERVzLByIiAqYRqOBjY1NUTfjqfXjjz/i+vXreOWVV/Lc3r59+3yvAHz33XfYu3cvPvvsMwwcOBAA8Pbbb+Oll17ClClT8MYbb8DNze2hj79+/ToAiArhtLQ02NnZGY0rroKCguDi4oLVq1dj8uTJRd0cIirmeKsSUSE7cuQINBoNvv/+e/266OhoaDQaPPPMMwax7du3R0BAQK4ce/bsQZMmTWBjY4MqVargiy++yBVz7tw5vPzyy3B1dUWZMmXQtGlT/PTTT6I2/vPPP3jppZfg6uoKGxsbNG7c2KC9DzNx4kSMGjUKAODr66u/beT8+fP6mK+++gqNGjWCra0tXF1d0atXL8THx+u3r1y5EhqNBp9//rlB7unTp0Oj0eDnn3/G+fPnUb58eQDApEmT9PvJ7378W7du4b333kPdunVhb28PR0dHtG/fHn/99ZdBXM7tLw+2GQB+++03aDSaXLdGLV++HFWrVoWtrS2aNGmC33//Pde+H9bH4ddff0XLli1hZ2cHZ2dndOnSBSdOnHjoc3hQeno6Jk6ciOrVq8PGxgYVKlRA9+7dcfbsWX1MWloaRo4cCS8vL1hbW6NGjRr46KOPoJQyyJWRkYERI0agfPnycHBwQOfOnXHp0qU893v58mUMHDgQ7u7usLa2Ru3atXOdKwBYtGgRateujTJlysDFxQWNGzfG2rVrRc/tvzZv3ozKlSvn+wv4nTt3oNVq89yWc04evM0p59/p6enYsmXLQ/NOnDgRPj4+AIBRo0ZBo9GgcuXK+m0ajQbHjx/Hq6++ChcXF7Ro0QLA/fd5//79UaVKFdjY2MDDwwMDBw7EzZs3c+XXaDQ4deoUXnvtNTg5OaF8+fIYN24clFKIj49Hly5d4OjoCA8PD8yZMydXGzMyMjBhwgRUq1YN1tbW8PLywujRo5GRkWEQt2PHDrRo0QLOzs6wt7dHjRo1cl1tsbS0ROvWrfM9JkREOVg4EBWyOnXqwNnZGVFRUfp1v//+O8zMzPDXX38hJSUFAKDT6bB37160atXK4PFnzpzBSy+9hBdeeAFz5syBi4sL+vfvj2PHjuljrl27hmbNmulvx5g2bRrS09PRuXNnbNq0Kd/2HTt2DE2bNsWJEyfw/vvvY86cObCzs0PXrl2NPrZ79+7o3bs3AGDevHn48ssv8eWXX+q/5E+bNg19+/aFn58f5s6di+HDhyMyMhKtWrXS94kYMGAAOnbsiPDwcH1BcfToUUyaNAmDBg1Chw4dUL58ef396d26ddPvp3v37g9t27lz57B582Z07NgRc+fOxahRo3D06FE899xzuHLlSr7P62E+++wzvPnmm/Dw8MCsWbPQvHlzdO7c2aAQepidO3ciODgYiYmJmDhxIsLDw7F37140b948V9HyX1qtFh07dsSkSZPQqFEjzJkzB++++y6Sk5P1HVuVUujcuTPmzZuHF198EXPnzkWNGjUwatQohIeHG+R74403MH/+fLRr1w4zZsyApaUlQkJCcu332rVraNq0KXbu3ImwsDAsWLAA1apVw6BBgzB//nx93IoVKzBs2DD4+/tj/vz5mDRpEho0aIADBw4YP6h52Lt3b66i+kFt2rSBo6MjypQpg86dO+P06dMG2zMyMmBubg4rKyuD9WXKlAFwv3B/mO7du2PevHkAgN69e+PLL780eK4A8PLLL+Pu3buYPn06Bg8eDOD+l/Rz585hwIABWLRoEXr16oV169ahQ4cOuQo3AOjZsyd0Oh1mzJiBgIAATJ06FfPnz8cLL7yAihUrYubMmahWrRree+89g88OnU6Hzp0746OPPkKnTp2waNEidO3aFfPmzUPPnj31cceOHUPHjh2RkZGByZMnY86cOejcuTP++OOPXG1p1KgR/v77b/1nERHRQykiKnQhISGqSZMm+n93795dde/eXZmbm6tffvlFKaVUTEyMAqC2bNmij/Px8VEAVFRUlH5dYmKisra2ViNHjtSvGz58uAKgfv/9d/26O3fuKF9fX1W5cmWl1WqVUkrFxcUpAGrlypX6uLZt26q6deuq9PR0/TqdTqeaNWum/Pz8jD632bNnKwAqLi7OYP358+eVubm5mjZtmsH6o0ePKgsLC4P1V69eVa6uruqFF15QGRkZqmHDhsrb21slJyfrY65fv64AqAkTJhhtk1JKpaen6593jri4OGVtba0mT56sX7dy5co8279r1y4FQO3atUsppVRmZqZyc3NTDRo0UBkZGfq45cuXKwDqueeeM9jPf49zgwYNlJubm7p586Z+3V9//aXMzMxU3759830un3/+uQKg5s6dm2ubTqdTSim1efNmBUBNnTrVYPtLL72kNBqNOnPmjFJKqdjYWAVAvfPOOwZxr776aq7jO2jQIFWhQgV148YNg9hevXopJycndffuXaWUUl26dFG1a9fO9zlIZWVlKY1GY/D6zrF+/XrVv39/tXr1arVp0yb14YcfqjJlyqhy5cqpixcv6uPmzJmT6/2glFLvv/++AqA6duyYbxtyzt/s2bMN1k+YMEEBUL179871mJxj8aCvv/461/s3J8eQIUP067Kzs1WlSpWURqNRM2bM0K+/ffu2srW1Vf369dOv+/LLL5WZmVmu57Zs2TIFQP3xxx9KKaXmzZunAKjr16/n+1yVUmrt2rUKgDpw4IDRWCIq3XjFgegJaNmyJWJiYpCWlgbg/q1HHTp0QIMGDfS3Vfz+++/QaDT6Wx9y+Pv7o2XLlvp/ly9fHjVq1MC5c+f0637++Wc0adLE4LH29vYYMmQIzp8/j+PHj+fZrlu3buHXX3/FK6+8gjt37uDGjRu4ceMGbt68ieDgYJw+fRqXL19+pOf83XffQafT4ZVXXtHnvXHjBjw8PODn54ddu3bpYz08PLB48WLs2LEDLVu2RGxsLD7//HM4Ojo+0r4BwNraGmZm9z/itFotbt68qb9dIyYmxuR8hw8fRmJiIt566y2DX7L79+8PJyenfB979epVxMbGon///nB1ddWvr1evHl544QX8/PPP+T5+48aNKFeuHIYOHZprm0ajAXD/NWBubo5hw4YZbB85ciSUUvjll1/0cQByxQ0fPtzg30opbNy4EZ06dYJSyuAcBgcHIzk5WX8cnZ2dcenSJRw6dCjf5yFx69YtKKXg4uKSa9srr7yClStXom/fvujatSumTJmCbdu24ebNm5g2bZo+7tVXX4WTkxMGDhyIHTt24Pz581i+fDmWLFkC4H7n58fx1ltv5Vpna2ur/+/09HTcuHEDTZs2BYA8X29vvPGG/r/Nzc3RuHFjKKUwaNAg/XpnZ+dc7/VvvvkGtWrVQs2aNQ3OyfPPPw8A+vdVTv+MLVu2QKfT5ft8co71jRs38o0jImLhQPQEtGzZEtnZ2di3bx9OnjyJxMREtGzZEq1atTIoHPz9/Q2+WAKAt7d3rnwuLi64ffu2/t8XLlxAjRo1csXVqlVLvz0vZ86cgVIK48aNQ/ny5Q2WCRMmAAASExOh1WqRkJBgsGRmZub7nE+fPg2lFPz8/HLlPnHiBBITEw3ie/XqhZCQEBw8eBCDBw9G27Zt881vjE6nw7x58+Dn5wdra2uUK1cO5cuXx5EjR5CcnGxyvpxj6OfnZ7De0tISVapUET32Yefoxo0b+qIyL2fPnkWNGjVgYfHw8SwuXLgAT09PODg45Mr/YBsuXLgAMzOzXP0H/tu269evIykpCcuXL891/gYMGAAA+nM4ZswY2Nvbo0mTJvDz80NoaGiet8SYQuVxe09eWrRogYCAAOzcuVO/zsPDA99//z0yMjLQrl07+Pr6YtSoUVi0aBGA+0X14/jviE7A/YLn3Xffhbu7O2xtbVG+fHl9XF6vt/++r52cnGBjY4Ny5crlWv/ge/306dM4duxYrnNSvXp1AP87Jz179kTz5s3xxhtvwN3dHb169cKGDRvyLCJyjnVOEUpE9DAcVYnoCWjcuDFsbGwQFRUFb29vuLm5oXr16mjZsiWWLFmCjIwM/P777+jWrVuux+Y1HCUg/2KVn5wvEe+99x6Cg4PzjKlWrRri4+NzfVnatWtXvsOi6nQ6aDQa/PLLL3k+h/9+ebt58yYOHz4M4P7EVDqdTn/F4FFMnz4d48aNw8CBAzFlyhS4urrCzMwMw4cPN/jy9LAvSw/reFta5Byj1157Df369cszpl69egDuFycnT57Ejz/+iK1bt2Ljxo1YsmQJxo8frx9CV8rV1RUajcbgy7IxXl5eOHnypMG6Vq1a4dy5czh69CjS0tJQv359fd+WnC/Zj+rBqws5XnnlFezduxejRo1CgwYNYG9vD51OhxdffDHPL+t5vSck73WdToe6deti7ty5ecZ6eXnp2xgVFYVdu3bhp59+wtatW7F+/Xo8//zz2L59u8G+co71f4sWIqL/YuFA9ARYWVnpR+Dx9vbW33rUsmVLZGRkYM2aNbh27VqujtFSPj4+ub44AfdHS8rZnpecX8otLS0RFBT00PyWlpbYsWOHwbr69esDePgX76pVq0IpBV9fX9EXtdDQUNy5cwcREREYO3Ys5s+fb9Cp19RfQ7/99lu0adMGn332mcH6pKQkgy9IObdp/HcCu/9epck5hqdPn9bfFgLcn3MgLi5OfzzykvPYh52jcuXK5TukZ9WqVXHgwAFkZWU9dG4IHx8f7Ny5E3fu3DG46vDf14CPjw90Op3+KkaO/7YtZ8QlrVab72sjh52dHXr27ImePXsiMzMT3bt3x7Rp0zB27FiThqa1sLBA1apVERcXJ37MuXPn9B3yH2Rubo4GDRro/51zVULyfExx+/ZtREZGYtKkSQaTqf2303ZBqFq1Kv766y+0bdvW6HvCzMwMbdu2Rdu2bTF37lxMnz4dH3zwAXbt2mVwDOLi4mBmZvbYBRURlXy8VYnoCWnZsiUOHDiAXbt26QuHcuXKoVatWpg5c6Y+5lF06NABBw8exL59+/Tr0tLSsHz5clSuXBn+/v55Ps7NzQ2tW7fGJ598gqtXr+banjOevY2NDYKCggyWnC/cOV94//vFu3v37jA3N8ekSZNyXR1RShkMU/ntt99i/fr1mDFjBt5//3306tULH374IU6dOqWPyRkRRzpDtbm5ea79fvPNN7n6bOTcsvPgyDVarRbLly83iGvcuDHKly+PZcuWGdymtWrVKqNtqlChAho0aIDVq1cbxP7999/Yvn07OnTokO/je/TogRs3buDjjz/OtS3nOXbo0AFarTZXzLx586DRaNC+fXsA0P//woULDeL+O3KQubk5evTogY0bN+pHbnpQzmsDQK4hR62srODv7w+lFLKysvJ9bnkJDAzUX3162D5z/Pzzz4iOjsaLL76Yb87r169j5syZqFevXoEXDjm/3v/39fbfY1oQXnnlFVy+fBkrVqzIte3evXv6W95u3bqVa3tOEfXfYVujo6NRu3Zto311iIh4xYHoCWnZsiWmTZuG+Ph4gwKhVatW+OSTT1C5cmVUqlTpkXK///77+Prrr9G+fXsMGzYMrq6uWL16NeLi4rBx48Z8b/lZvHgxWrRogbp162Lw4MGoUqUKrl27hn379uHSpUu55j34r0aNGgEAPvjgA/Tq1QuWlpbo1KkTqlatiqlTp2Ls2LE4f/48unbtCgcHB8TFxWHTpk0YMmQI3nvvPSQmJuLtt99GmzZt9LP0fvzxx9i1axf69++PPXv2wMzMDLa2tvD398f69etRvXp1uLq6ok6dOqhTp06e7erYsSMmT56MAQMGoFmzZjh69CjWrFmTqz9C7dq10bRpU4wdOxa3bt2Cq6sr1q1bh+zsbIM4S0tLTJ06FW+++Saef/559OzZE3FxcVi5cqXRPg4AMHv2bLRv3x6BgYEYNGgQ7t27h0WLFsHJySnf+SgAoG/fvvjiiy8QHh6OgwcPomXLlkhLS8POnTvxzjvvoEuXLujUqRPatGmDDz74AOfPn0f9+vWxfft2bNmyBcOHD9cXSA0aNEDv3r2xZMkSJCcno1mzZoiMjMSZM2dy7XfGjBnYtWsXAgICMHjwYPj7++PWrVuIiYnBzp079V9O27VrBw8PDzRv3hzu7u44ceIEPv74Y4SEhOTqcyHRpUsXfPnllzh16pTBr+DNmjVDw4YN0bhxYzg5OSEmJgaff/45vLy8cs1P8NxzzyEwMBDVqlVDQkICli9fjtTUVPz444+PdQtcXhwdHdGqVSvMmjULWVlZqFixIrZv327SVROp119/HRs2bMBbb72FXbt2oXnz5tBqtfjnn3+wYcMGbNu2DY0bN8bkyZMRFRWFkJAQ+Pj4IDExEUuWLEGlSpUMBlHIysrC7t278c477xR4W4moBHrSwzgRlVYpKSnK3NxcOTg4qOzsbP36r776SgFQr7/+eq7H+Pj4qJCQkFzrn3vuOYPhP5VS6uzZs+qll15Szs7OysbGRjVp0kT9+OOPBjF5DROa89i+ffsqDw8PZWlpqSpWrKg6duyovv32W9FzmzJliqpYsaIyMzPLNbTpxo0bVYsWLZSdnZ2ys7NTNWvWVKGhoerkyZNKqftD0zo4OKjz588b5NyyZYsCoGbOnKlft3fvXtWoUSNlZWVldGjW9PR0NXLkSFWhQgVla2urmjdvrvbt2/fQYxcUFKSsra2Vu7u7+r//+z+1Y8cOg+FYcyxZskT5+voqa2tr1bhxYxUVFZUr58OO886dO1Xz5s2Vra2tcnR0VJ06dVLHjx83enyVuj/c5wcffKB8fX2VpaWl8vDwUC+99JI6e/asPubOnTtqxIgRytPTU1laWio/Pz81e/Zs/ZCtOe7du6eGDRumypYtq+zs7FSnTp1UfHx8nsf02rVrKjQ0VHl5een327ZtW7V8+XJ9zCeffKJatWqlypYtq6ytrVXVqlXVqFGjDIbTNUVGRoYqV66cmjJlisH6Dz74QDVo0EA5OTkpS0tL5e3trd5++22VkJCQK8eIESNUlSpVlLW1tSpfvrx69dVXDY5VfowNx5rXEKeXLl1S3bp1U87OzsrJyUm9/PLL6sqVK7mO6cNy9OvXT9nZ2eXK+9xzz+Ua6jYzM1PNnDlT1a5dW1lbWysXFxfVqFEjNWnSJP0xj4yMVF26dFGenp7KyspKeXp6qt69e6tTp04Z5Prll18UAHX69GnRsSGi0k2jVAH0sCQiIipAU6ZMwcqVK3H69OmHdhqmx9e1a1doNBqjkz0SEQEACwciIip2UlNTUaVKFcybNw99+vQp6uaUSCdOnEDdunURGxv70Fv+iIgexMKBiIiIiIiM4qhKRERERERkFAsHIiIiIiIyioUDEREREREZxcKBiIiIiIiMKvETwOl0Oly5cgUODg7QaDRF3RwiIiIi+g+lFO7cuQNPT88Cn6SxIKSnpyMzM1MUa2VlBRsbm0JuUdEo8YXDlStX4OXlVdTNICIiIiIj4uPjUalSpaJuhoH09HT4+tgjIVErivfw8EBcXFyJLB5KfOHg4OAAAKj+xniYWxk/gWleOlFeu4vyaviep2zEW+9t6eKcl4dki+Ky4+3EOR3Oya7IJNWX7RsA3HfLJm663licEkp66F1kvwwAgKV1lijOcbu9OGfZbedEccrDVZzTfu4NUdzVVEdxzpt3yohjfafcE8XdblROnPO1UT+L4tZOay/OaSb7bIf3sNPinKdX1RDFWWTKR7i2i5e9599ZulGcc8XzAaK4jLo+4pzmGbIDanHqkjjntW5+orgMF/mVYs/f00Rxd8ekinP29DosivtmYrA454268j+91kmyONvrsr9dAPDOB9+K4pbMeEmc81pL2WvE6YilOKfHxlOiuJOTfMU5zVJlf5O8t8r+JgBASmUrUVxaBflr2eGi7HMky16es/yfstf9hY7y7w1lLsn2fzfQ+HtTdy8DF0I/0n9vK04yMzORkKhFXLQPHB3y/xKSckcH30YXkJmZKSocoqKiMHv2bERHR+Pq1avYtGkTunbtqt+ulMKECROwYsUKJCUloXnz5li6dCn8/P73GXrr1i0MHToUP/zwA8zMzNCjRw8sWLAA9vb/+85y5MgRhIaG4tChQyhfvjyGDh2K0aNHm3wsSnzhkHN7krmVDcytjZ9AMxvZh6+5tbxwMLORfQBYmHA2zMvIvrzrTKh2za1kHwBmtvLCwcJS9iFtZkJRrqSTyNrKz5G5jSyppPjMYWEm+2OizK3FOS3tZDktlDynudaE52QufH+YcJxs7WUvfAtLeU7pVW7p8QTkz8nchKlxpO/5Mg7ymZOlrzuthQmfDVph4SDcN2DC8bSWfzGysBC2007+pbAwXp/m1vIPe+nHg4WlvHCQvp7MTXnP2cqOvbmVvHCQvp7MbE1oZ7bsuVtYyN9z5laydpryWja3kn2O6IR/twHAwkL2t9usML43lBH+mgMU69vKbe0VbO3zPzdZJk6PlpaWhvr162PgwIHo3r17ru2zZs3CwoULsXr1avj6+mLcuHEIDg7G8ePH9YVJnz59cPXqVezYsQNZWVkYMGAAhgwZgrVr1wIAUlJS0K5dOwQFBWHZsmU4evQoBg4cCGdnZwwZMsSk9ha/m8jysHjxYlSuXBk2NjYICAjAwYMHi7pJRERERFSK6IT/M0X79u0xdepUdOvWLdc2pRTmz5+PDz/8EF26dEG9evXwxRdf4MqVK9i8eTOA+zPAb926FZ9++ikCAgLQokULLFq0COvWrcOVK1cAAGvWrEFmZiY+//xz1K5dG7169cKwYcMwd+5ck49BsS8c1q9fj/DwcEyYMAExMTGoX78+goODkZiYWNRNIyIiIqJSQquUaAHu/8r/4JKRkWHy/uLi4pCQkICgoCD9OicnJwQEBGDfvn0AgH379sHZ2RmNG//vnu+goCCYmZnhwIED+phWrVrB6oGrY8HBwTh58iRu375tUpuKfeEwd+5cDB48GAMGDIC/vz+WLVuGMmXK4PPPPy/qphERERFRKaGDEi0A4OXlBScnJ/0SERFh8v4SEhIAAO7u7gbr3d3d9dsSEhLg5uZmsN3CwgKurq4GMXnleHAfUsW6j0NmZiaio6MxduxY/TozMzMEBQXpK63/ysjIMKjqUlJSCr2dRERERFSy6aCgRf59GHIKh/j4eDg6/m+gEmtred/D4qxYX3G4ceMGtFptvpXWf0VERBhUeByKlYiIiIgelylXHBwdHQ2WRykcPDw8AADXrl0zWH/t2jX9Ng8Pj1y372dnZ+PWrVsGMXnleHAfUsW6cHgUY8eORXJysn6Jj48v6iYRERER0VPOlD4OBcHX1xceHh6IjIzUr0tJScGBAwcQGBgIAAgMDERSUhKio6P1Mb/++it0Oh0CAgL0MVFRUcjK+t+Icjt27ECNGjXg4uJiUpuKdeFQrlw5mJub51tp/Ze1tXWuKo+IiIiI6HHohIspUlNTERsbi9jYWAD3O0THxsbi4sWL0Gg0GD58OKZOnYrvv/8eR48eRd++feHp6amf66FWrVp48cUXMXjwYBw8eBB//PEHwsLC0KtXL3h6egIAXn31VVhZWWHQoEE4duwY1q9fjwULFiA8PNzkY1CsCwcrKys0atTIoNLS6XSIjIzUV1pERERERIVN+28fB2OLKQ4fPoyGDRuiYcOGAIDw8HA0bNgQ48ePBwCMHj0aQ4cOxZAhQ/Dss88iNTUVW7duNZhcbs2aNahZsybatm2LDh06oEWLFli+fLl+u5OTE7Zv3464uDg0atQII0eOxPjx402ewwEo5p2jgfsHsF+/fmjcuDGaNGmC+fPnIy0tDQMGDCjqphERERFRKZGl7i/GYkzRunVrqHxub9JoNJg8eTImT5780BhXV1f9ZG8PU69ePfz++++mNS4Pxb5w6NmzJ65fv47x48cjISEBDRo0wNatW3N1mCYiIiIiKiw6aKBF/jNb64xsf9oV+8IBAMLCwhAWFlbUzSAiIiKiUkqn7i/GYkqyp6JwKAjO7a7Cws74UFiZezxF+Spulc9cnVrTVRSX4mNjPOhfHs5XRHG3/nAQ5zTLlr3anWMtxTkzXrshirM+WE6c0+6SsJ2n5V2ULE9cFsUt//N7cc7Xro8QxSkz+a8TaWmymSdbe5wW5/zDvIo49kpwRVFcVutkcc6VkzuL4hKbiVNCCXtvdXaUj7p2JrOmKM439KQ455/baonipv9ff3FOZ8s4UdzV5vKhASu2lh2nezr55439Yq0ozrrLTXHOWYPzv1Sfo+fBweKcM/e1F8WZt5J3GdQ6ZhkP+pfOSvZn2jJV/jnywXeviuL8Is+KcyYEeYvi3F+6IM6ZdUSWs8rX8s96naXs78elN+XnyPyI7L2U7p0pzukQIPsMTT0q/9tp2eGeKE79bSfOmeotO54fNPzFaMy91GwU95+ItYIrDsa2P+1KTeFARERERPSoWDiwcCAiIiIiMkqnNNApI30cjGx/2rFwICIiIiIyglccWDgQERERERmlhRm0RqZAk/XeenqxcCAiIiIiMkIJblVSvFWJiIiIiKh0y1TmsDQybF8mCwciIiIiotJNBw10Rm5V0qFkT+TAwoGIiIiIyAh2jmbhQERERERklFaZQWvkViWt4hUHIiIiIqJS7f6tSkbmceAVByIiIiKi0k0nGI6VfRyIiIiIiEo53qrEwoGIiIiIyCgdzDiqUlE3gIiIiIiouNMqDbRG5mkwtv1pV2oKB+tpDrCwsDEa5+6aJcqXVt1VvO/0IbdFcbeS7MU5r5/yEMVVPp4pzml1O0MUd6GDgzin9zTjxxwAys89Ls45osJ2UdzMy+3FOd+ruE0UN+ilt8U5NcKXSP85m8U5p/zcXRTXu8Zacc4/xgSIY9Payn5JcbO7J86Z9ZrsNeqxSv6eczh/VxS3pUZ9cc6kLmmiuCvTq4lzlrfIFsUtnL9InHPAnBGiOI8D8s8G7CorCitz/ro4ZWoDWdy8GhvEOT9o1UMU5/KJ7PUBAGVWWIridJZaec4LyeLYtCrOorgbdeV/zi2ET39LzC/inKk62d+PFkveE+f0m3ZWFGdhJj/2F5JlnyMNXW6Ic65uIfv7YWbkl+oHNY/tJYqrvvSSOOf5V71kOddfEeeM6+Mpipse+6LRGN3ddAAHxfsuClnKAlnK3EgMCwciIiIiolJNK+gcreWtSkREREREpZsOxm9F0j2ZphQZ+XWzIhAREYFnn30WDg4OcHNzQ9euXXHy5MmibhYRERERlTI5naONLSVZsX52u3fvRmhoKPbv348dO3YgKysL7dq1Q1qa7F5jIiIiIqKCkDMcq7GlJCvWtypt3brV4N+rVq2Cm5sboqOj0apVqyJqFRERERGVNpw5upgXDv+VnHx/JApX14ePipCRkYGMjP+N7pCSklLo7SIiIiKikk02AVzJvuLw1Dw7nU6H4cOHo3nz5qhTp85D4yIiIuDk5KRfvLxkw48RERERET1MzqhKxpaS7Kl5dqGhofj777+xbt26fOPGjh2L5ORk/RIfH/+EWkhEREREJZVOaURLSfZU3KoUFhaGH3/8EVFRUahUqVK+sdbW1rC2tn5CLSMiIiKi0kAnuKJQ0kdVKtaFg1IKQ4cOxaZNm/Dbb7/B19e3qJtERERERKVQljKHudGZozkBXJEJDQ3F2rVrsWXLFjg4OCAhIQEA4OTkBFtb2yJuHRERERGVFjplBp2Rzs/Gtj/tivWzW7p0KZKTk9G6dWtUqFBBv6xfv76om0ZEREREpYgWgBYaI0vJVqyvOKgSfrmHiIiIiJ4OvOJQzAuHgpRe1gYWljZG4y63lp3w8jHyfX/sv1YU12ffYHHOdR0+FsUdfV4+HO3asBBRnM+PyeKc5RZdFsWt9IkU5zySKYuzMc8S55zUsqsoruO2KHHOA8myPjlr3uggzhkw86Qobspl2bkEADXyhjjW/p7x9xAAOL+jE+dMauwminM4L58x/uzL9qK4quPEKeH60XVRnP3/ZRgP+telWX6iuNf+HCjOWWmf7P155lUHcU6XGrL5cMbV2CXOuWjAK6K4Me++Lc756/5lorhWR18S57zWRDbYRkZZ+Wve6WRZcazjy1dEceVNGMnlbpalKK52lPx15+x4VxQ3+LWfxTmXn2ghijMzk//I6DVZFntphrM4Z7/zwaK4bJ38C2VSShlR3I4/vhDnDD7yuiju6u0K4pyRg2eJ4jpOHWU0RpuZf9+B4oDzOJSiwoGIiIiI6FEpwczRijNHExERERGVbrziUMw7RxMRERERFQeFMQGcVqvFuHHj4OvrC1tbW1StWhVTpkwx6OerlML48eNRoUIF2NraIigoCKdPnzbIc+vWLfTp0weOjo5wdnbGoEGDkJqaWiDP+0EsHIiIiIiIjND+OwGcscUUM2fOxNKlS/Hxxx/jxIkTmDlzJmbNmoVFixbpY2bNmoWFCxdi2bJlOHDgAOzs7BAcHIz09HR9TJ8+fXDs2DHs2LFDP2nykCFDCuy55+CtSkRERERERmQLJoDLVvKBEgBg79696NKlC0JC7g9qUrlyZXz99dc4ePAggPtXG+bPn48PP/wQXbp0AQB88cUXcHd3x+bNm9GrVy+cOHECW7duxaFDh9C4cWMAwKJFi9ChQwd89NFH8PT0NPWpPhSvOBARERERGaFVGtECACkpKQZLRkbeI+41a9YMkZGROHXqFADgr7/+wp49e9C+fXsAQFxcHBISEhAUFKR/jJOTEwICArBv3z4AwL59++Ds7KwvGgAgKCgIZmZmOHDgQIEeA15xICIiIiIyQtKHIWe7l5fhcPgTJkzAxIkTc8W///77SElJQc2aNWFubg6tVotp06ahT58+AICEhAQAgLu7u8Hj3N3d9dsSEhLg5mY4tLmFhQVcXV31MQWFhQMRERERkRFKMAGc+nd7fHw8HB0d9eutrfOeE2bDhg1Ys2YN1q5di9q1ayM2NhbDhw+Hp6cn+vXrV3CNLyAsHIiIiIiIjNBCA62ReRpytjs6OhoUDg8zatQovP/+++jVqxcAoG7durhw4QIiIiLQr18/eHh4AACuXbuGChX+NznftWvX0KBBAwCAh4cHEhMTDfJmZ2fj1q1b+scXFPZxICIiIiIyQqckQ7KalvPu3bswMzP8Om5ubg6d7n4na19fX3h4eCAyMlK/PSUlBQcOHEBgYCAAIDAwEElJSYiOjtbH/Prrr9DpdAgICHjEZ5s3k684XLp0CZUqVcpz2/79+9G0adPHbhQRERERUXGiE9yqZGz7f3Xq1AnTpk2Dt7c3ateujT///BNz587FwIEDAQAajQbDhw/H1KlT4efnB19fX4wbNw6enp7o2rUrAKBWrVp48cUXMXjwYCxbtgxZWVkICwtDr169CnREJeARCod27dphz549cHV1NVj/xx9/ICQkBElJSQXVNiIiIiKiYkEHDXRGblUytv2/Fi1ahHHjxuGdd95BYmIiPD098eabb2L8+PH6mNGjRyMtLQ1DhgxBUlISWrRoga1bt8LGxkYfs2bNGoSFhaFt27YwMzNDjx49sHDhQtOeoIDJhUPTpk3Rrl077Nq1Cw4ODgCAqKgodOrUKc/e4kRERERET7sHh1vNL8YUDg4OmD9/PubPn//QGI1Gg8mTJ2Py5MkPjXF1dcXatWtN2vejMLmPw6effgpvb2906tQJGRkZ2LVrF0JCQjB58mSMGDGiMNpIRERERFSkspU5snVGFiMTxD3tTL7iYGZmhnXr1iEkJATPP/88jhw5goiICISFhRVG+wqM3ZlbsDDPeyisB61f9LUoX6/sd8X7rmGZLYpb1ES2bwAYd66bKO7qz97inDtXzRbFvTRUXiBev2cviqv1dag4Z9WGl0RxZ//Muy9OXip9Lhvn+McBrcQ5z/WQPXfL4XfEOd01sl5Xh6JqinP+0ecjcaz03P+0Z50455o7ZUVxHy3oKc5Zru41UdxZnZvxoH9ZRDqI4kb03izOufGq7DU6tvZWcc7lnj1Ece61E40H/evWXtmoHPMXvirOeamDpSjO3C9VnLNjyGuiuGuvGR/pJEe5FrLjVPnNdHHOk+/KP5sc5sleo/ajZZ+LAHAluoLxIAAV9mrFOXd/sloUV+uTd8Q5zbJkcT5fXxbnPPWm7H7v0/Xkv9z675W97irOl3/lqizsYNvli2HinM43ZK9R8xtXxDn7/i377lc+LcVoTLY27wnSihMluFVJmXir0tNG9Co+cuRIrnUTJ05E79698dprr6FVq1b6mHr16hVsC4mIiIiIipgpE8CVVKLCoUGDBtBoNFDqfyVwzr8/+eQTLF++HEopaDQaaLXyXyiIiIiIiJ4GhTGq0tNGVDjExcUVdjtEZsyYgbFjx+Ldd9/NtxMJEREREVFB4hUHYeHg4+NT2O0w6tChQ/jkk094KxQRERERPXGFMRzr0+aRrqecPXsWQ4cORVBQEIKCgjBs2DCcPXu2oNuml5qaij59+mDFihVwcXEptP0QEREREeXF+KzRxq9IPO1MLhy2bdsGf39/HDx4EPXq1UO9evVw4MAB1K5dGzt27CiMNiI0NBQhISEICgoyGpuRkYGUlBSDhYiIiIjocbBweIThWN9//32MGDECM2bMyLV+zJgxeOGFFwqscQCwbt06xMTE4NChQ6L4iIgITJo0qUDbQERERESlG/s4PMIVhxMnTmDQoEG51g8cOBDHjx8vkEbliI+Px7vvvos1a9YYTKudn7FjxyI5OVm/xMfHF2ibiIiIiKj00SoNspVZvoupM0c/bUy+4lC+fHnExsbCz8/PYH1sbCzc3OSTKUlER0cjMTERzzzzjH6dVqtFVFQUPv74Y2RkZMDc3HCGPmtra1hbG5/ojYiIiIhIilccHqFwGDx4MIYMGYJz586hWbNmAIA//vgDM2fORHh4eIE2rm3btjh69KjBugEDBqBmzZoYM2ZMrqKBiIiIiKgwsHB4hMJh3LhxcHBwwJw5czB27FgAgKenJyZOnIhhw+RTn0s4ODigTp06Buvs7OxQtmzZXOuJiIiIiAoLC4dHKBw0Gg1GjBiBESNG4M6dOwDuf8EnIiIiIiqpWDg8QuGQ4/r16zh58iQAoGbNmihXrlyBNSo/v/322xPZDxERERFRDqU0UEYKA2Pbn3YapZQy5QFpaWkYOnQovvjiC+h0OgCAubk5+vbti0WLFqFMmTKF0tBHlZKSAicnJ1QfMR3m1sZHZrJ/LlGU12aRfCK6G/UsRXFZ9vJTUeaa7IX51XtzxDnHXugminvPa6s455ur3xHFZTnoxDmbNZeN3hV7raI4p+tye1Gc2/+dE+ec6bVFFNdpyWhxzgwX2WtEWcpfS1Y+qeLYzAuy42QK65uywd0yneTPSWfC85eyvyBrZ7adPKel8NC7HU4T57z9f/dEcdYW2eKcK2p+JYrrvH6kOOc/ry0Wxa1LLS/OuSmxoSgu8aMq4pwa4UfTlZbyPnd+Ef+IY+1/kL3urMy04pz7/6gliovtPU+c095MNvLhbe1dcc5ePd4UxQ366ntxzta2V2T7HvSuOOei5YtEce+EyXPa/XNdFpgs//zOqllJFGd+L0ucU/P3GVHctQ0+RmO0dzPwd8+PkJycDEdHR3EbnoSc75KBW4bCwi7/AXiy0zKwr8uiYvk8CoLJw7GGh4dj9+7d+OGHH5CUlISkpCRs2bIFu3fvxsiR8j8aRERERERPC04A9wi3Km3cuBHffvstWrdurV/XoUMH2Nra4pVXXsHSpUsLsn1EREREREWOtyo9QuFw9+5duLu751rv5uaGu3fllyCJiIiIiJ4W7Bz9CLcqBQYGYsKECUhPT9evu3fvHiZNmoTAwMACbRwRERERUXGg05lBa2TR6Uz+av1UMfmKw4IFCxAcHIxKlSqhfv36AIC//voLNjY22LZtW4E3kIiIiIioqCkAxoYUKvihOYoXkwuHOnXq4PTp01izZg3++ef+yBC9e/dGnz59YGtrW+ANJCIiIiIqajpooIGRW5WMbH/aPdI8DmXKlMHgwYMLui1ERERERMUSO0c/YuFw8uRJLFq0CCdOnAAA1KpVC2FhYahZs2aBNo6IiIiIqDjQKQ007Bxtmo0bN6JOnTqIjo5G/fr1Ub9+fcTExKBu3brYuHFjYbSRiIiIiKhIKSVbSjKTrziMHj0aY8eOxeTJkw3WT5gwAaNHj0aPHj0KrHFERERERMUBb1V6hCsOV69eRd++fXOtf+2113D16tUCaRQRERERUXGSUzgYW0oykwuH1q1b4/fff8+1fs+ePWjZsmWBNIqIiIiIqDjJmQDO2FKSiW5V+v777/X/3blzZ4wZMwbR0dFo2rQpAGD//v345ptvMGnSpMJpJRERERFREdLpAI3OSOdo3RNqTBERFQ5du3bNtW7JkiVYsmSJwbrQ0FC89dZbBdIwIiIiIqLign0chIWDrgSUT5nOCmY2xru6W28uL8p3vU+6eN/VZ90RxdktviHO6WaTKoob0VteyN3xlU3gFz+hrDin4znZ8AJTJnwmzvnhya6iuDu3y4hzZjeyFMW5mjCVvPRdU+5Iljjn7eqydqa7iVNibSP5sX8/YqAo7kwvR3FO69uy14jdZXFKo8Pl5VgxeZ445yjfQFHcxXGyOADw+CRaFJfZso4459295URxZpfkQ3+MWN9GFLfxpPx4Bsa+LoorPyhFnPOrQ5tEccFD+olzZmTJxhBZV3+1OOfbtfuIY5GVJgqbX+Ubccr3A2SfI8Hhw8U57S7L/iamVbQR57zeVfY+fv+X3uKcFX+Tve41tvL3x4Bjuft95qXMsERxzss/eYri7nnI21ntswRRnPPqJHHOlCwXUVzWz8a/N2gz5N+rioqC8ZmhS/igSqb3cXhQenrxP8lERERERI+LnaMfoXDQarWYMmUKKlasCHt7e5w7dw4AMG7cOHz2mfyXSyIiIiKip4YSLiWYyYXDtGnTsGrVKsyaNQtWVlb69XXq1MGnn35aoI0DgMuXL+O1115D2bJlYWtri7p16+Lw4cMFvh8iIiIiooeSXG3gFQdDX3zxBZYvX44+ffrA3Nxcv75+/fr4559/CrRxt2/fRvPmzWFpaYlffvkFx48fx5w5c+DiIrunjoiIiIioIBTWzNHGfiRXSmH8+PGoUKECbG1tERQUhNOnTxvkuHXrFvr06QNHR0c4Oztj0KBBSE2V9Yc1hckzR1++fBnVqlXLtV6n0yErS97JU2LmzJnw8vLCypUr9et8fX0LdB9ERERERMYUxqhKOT+St2nTBr/88gvKly+P06dPG/xIPmvWLCxcuBCrV6+Gr68vxo0bh+DgYBw/fhw2NvcHHOjTpw+uXr2KHTt2ICsrCwMGDMCQIUOwdu1a059oPky+4uDv75/nBHDffvstGjZsWCCNyvH999+jcePGePnll+Hm5oaGDRtixYoV+T4mIyMDKSkpBgsRERER0WPJuRXJ2GKCB38kb9KkCXx9fdGuXTtUrVr1/i6Vwvz58/Hhhx+iS5cuqFevHr744gtcuXIFmzdvBgCcOHECW7duxaeffoqAgAC0aNECixYtwrp163DlypUCPQQmFw7jx49HWFgYZs6cCZ1Oh++++w6DBw/GtGnTMH78+AJt3Llz57B06VL4+flh27ZtePvttzFs2DCsXv3wYe8iIiLg5OSkX7y8vAq0TURERERU+iidbAGQ60fsjIyMPHMa+5E8Li4OCQkJCAoK0q9zcnJCQEAA9u3bBwDYt28fnJ2d0bhxY31MUFAQzMzMcODAgQI9BiYXDl26dMEPP/yAnTt3ws7ODuPHj8eJEyfwww8/4IUXXijQxul0OjzzzDOYPn06GjZsiCFDhmDw4MFYtmzZQx8zduxYJCcn65f4+PgCbRMRERERlT6mDMfq5eVl8EN2REREnjmN/UiekHB//g13d3eDx7m7u+u3JSQkwM3NcAInCwsLuLq66mMKisl9HACgZcuW2LFjR4E2JC8VKlSAv7+/wbpatWph48aND32MtbU1rK2tC7tpRERERFTaCDs/x8fHw9Hxf5OhPuy7qU6nQ+PGjTF9+nQAQMOGDfH3339j2bJl6NdPPlnlk/JYE8AVtubNm+PkyZMG606dOgUfH58iahERERERlUamXHFwdHQ0WB5WODzsR/KLFy8CADw8PAAA165dM4i5du2afpuHhwcSEw1nJs/OzsatW7f0MQVFdMXBxcUFGo2ss8etW7ceq0EPGjFiBJo1a4bp06fjlVdewcGDB7F8+XIsX768wPZBRERERGSUZII3E4djNfYjua+vLzw8PBAZGYkGDRoAuN9/4sCBA3j77bcBAIGBgUhKSkJ0dDQaNWoEAPj111+h0+kQEBBgWoOMEBUO8+fP1//3zZs3MXXqVAQHByMwMBDA/U4Z27Ztw7hx4wq0cc8++yw2bdqEsWPHYvLkyfD19cX8+fPRp0+fAt0PEREREVH+NP8uxmLkjP1IrtFoMHz4cEydOhV+fn764Vg9PT3RtWtXAPevULz44ov6fsBZWVkICwtDr1694OnpafrTzIeocHjwHqsePXpg8uTJCAsL068bNmwYPv74Y+zcuRMjRowo0AZ27NgRHTt2LNCcREREREQmKYQrDpIfyUePHo20tDQMGTIESUlJaNGiBbZu3aqfwwEA1qxZg7CwMLRt2xZmZmbo0aMHFi5caFpjBDRKmTbHnb29PWJjY3NNAnfmzBk0aNCgUGapexwpKSlwcnJC3Q0jYV7GeKfplp7nRHlP9ZQP85rt7iSKOz1I3lfd4oalKK7ib9ninJfayPbvUfea8aB/Xb7iKoprWO2COOffe3NPQJgX5X1PnLO+1yVRXNxXfuKcd91lvzq4xcjPUcLr6aI4nVbefclvSpo49vQ4O1Fcdprs9QkANZbdFcXdquNoPOhfmY6yY3+vhfzzqup4WTtPjHEW56w1O1kUd26SjfGgf1V3vy6KO7OjijhnpotOFOduwqh/TkdviuLiXi4vzlnmmuzP2a2GWnFOcwfZxKbaTPl7zvacfAAPa9lhQobsoxYAULnteVHche2VxTmlxz7VS/5rbOVNslufrzwvf/K1Xz4hirv9ovxz+fTE2qK4SnXlo9skJDmI4sLrRopzzvqznShuSZM14pzDYnqJ4nQ64+dddzcdcQOmIzk52aBTcXGQ813Sa8lEmNnm/3msu5eO+HcmFsvnURBM7hxdtmxZbNmyJdf6LVu2oGzZsgXSKCIiIiKiYqUQJoB72pg8HOukSZPwxhtv4LffftN3uDhw4AC2bt1qdFZnIiIiIqKn0YMTvOUXU5KZXDj0798ftWrVwsKFC/Hdd98BuN8pY8+ePQXec5uIiIiIqFiQXFHgFYfcAgICsGaN/B44IiIiIqKnmUbdX4zFlGSPVDhotVps3rwZJ07c72RUu3ZtdO7cGebm5gXaOCIiIiKiYqEQRlV62phcOJw5cwYhISG4dOkSatSoAQCIiIiAl5cXfvrpJ1StWrXAG0lEREREVKR4q5LpoyoNGzYMVapUQXx8PGJiYhATE4OLFy/C19cXw4YNK4w2EhEREREVLSVcSjCTrzjs3r0b+/fvh6vr/8ZNLlu2LGbMmIHmzZsXaOOIiIiIiIoF3qpkeuFgbW2NO3fu5FqfmpoKKyurAmkUEREREVGxwsLB9FuVOnbsiCFDhuDAgQNQSkEphf379+Ott95C586dC6ONRERERERFixPAmV44LFy4EFWrVkVgYCBsbGxgY2OD5s2bo1q1aliwYEFhtJGIiIiIqEjlDMdqbCnJTL5VydnZGVu2bMGZM2f0w7HWqlUL1apVK/DGEREREREVC7xV6dHmcQCAatWqsVggIiIiolJBA8EEcE+kJUXH5MKhR48eaNKkCcaMGWOwftasWTh06BC++eabAmtcQXL+2A4WFjZG446a1Rfls7JJE+/7wy9XieL6bR8izule75oobsur8hm+B8V1EsU5WaaLc1osLCuKOx7gJ85pmSGLM7PNFOeUut1MuHMA9k73RHFNu50Q5wxxihXFfXylrTjnoeFVxLHh9beL4n4c0Eqcc+DXP4ri3MxzD8rwMAfuyuaTWb3uBXHO8b+sFMX1/u1Ncc7aa86I4uaX3SPOOazqc6K49E8rinO+XD9aFPf3Rx7inPFLXURxr/juFufcP/gZUVzV16+Icz7rfF4U94ytLA4APprzsjj2hXUHRXEXM1yNB/1rx8Ymorh7HjpxzsGvbRXHSr36xjFRXJuD8r+dzV1k77kTv3qKc5qnXRDFPVfulDinf5XLorhzGe7inG/W+10U18A6SZxz/jMbRHGZyvgEwXfvaNFHvOciwnkcTO/jEBUVhQ4dOuRa3759e0RFRRVIo4iIiIiIihXO42D6FYeHDbtqaWmJlJSUAmkUEREREVGxwj4Opl9xqFu3LtavX59r/bp16+Dv718gjSIiIiIiKk44qtIjXHEYN24cunfvjrNnz+L5558HAERGRuLrr78u8P4NWq0WEydOxFdffYWEhAR4enqif//++PDDD6HRlOx7yIiIiIioGOEVB9MLh06dOmHz5s2YPn06vv32W9ja2qJevXrYuXMnnntO1ilPaubMmVi6dClWr16N2rVr4/DhwxgwYACcnJwwbNiwAt0XEREREdFDsXB4tOFYQ0JCEBISUtBtyWXv3r3o0qWLfl+VK1fG119/jYMHHz7KREZGBjIy/jfyDftdEBEREdHjktyKVNJvVTK5j8OT1KxZM0RGRuLUqftDmP3111/Ys2cP2rdv/9DHREREwMnJSb94eXk9qeYSERERUUml08iWEuyRJ4B7Et5//32kpKSgZs2aMDc3h1arxbRp09Cnz8NH+h07dizCw8P1/05JSWHxQERERESPhVccinnhsGHDBqxZswZr165F7dq1ERsbi+HDh8PT0xP9+vXL8zHW1tawtrZ+wi0lIiIiohKNfRyKd+EwatQovP/+++jVqxeA+0PBXrhwAREREQ8tHIiIiIiICpxkuFUWDkXn7t27MDMz7IZhbm4OnU5XRC0iIiIiolKJVxxkhcODfQaMmTt37iM35r86deqEadOmwdvbG7Vr18aff/6JuXPnYuDAgQW2DyIiIiIio1g4yAqHP//80+DfMTExyM7ORo0aNQAAp06dgrm5ORo1alSgjVu0aBHGjRuHd955B4mJifD09MSbb76J8ePHF+h+iIiIiIjyw87RwsJh165d+v+eO3cuHBwcsHr1ari4uAAAbt++jQEDBqBly5YF2jgHBwfMnz8f8+fPL9C8RERERERkGpP7OMyZMwfbt2/XFw0A4OLigqlTp6Jdu3YYOXJkgTawoFgmZcDC3HicJksrync9wFW875nPdxbF1fj8sjjnt9U3iuL6npPt2xRvuu8yHvSvcSOdRXFrqnwnztl7/2BRnDrtJM55b0SqKG7+jnXinBMW9BfF1Qq7Is75xrq3RXH2F8UpYVFZHrth3IuiuB4rt4lzJmntRHFRyTXFOTu7xIjiGg/8RJzz11R/UVy7OsfEOS01ss+bYdWfF+c8P/4ZUVy3ugfEOYeW3SOKcz4k/5PyVUpVUZyn5W1xzjZfnxDFjZr2pjjnnWg3UVykWVNxzuGbvxXHBtokieKarJTfUvzH27NFcW0WjRLnPHPXXRR3apj8fZy1QvBHG8Dd67LPEACw8c8SxQ1z+1WcM0lnJYqb0LWvOKf/xk2iuC4O8s+bo5nlRHFJJnQj9be6KYrbe8/4sPh3lezzsEjxViXTC4eUlBRcv3491/rr16/jzp07BdIoIiIiIqLiRKMAjZHCqqTfqmTyzNHdunXDgAED8N133+HSpUu4dOkSNm7ciEGDBqF79+6F0UYiIiIioqKlhEsJZvIVh2XLluG9997Dq6++iqys+5f8LCwsMGjQIMyeLbv8SURERET0NGHn6EcoHMqUKYMlS5Zg9uzZOHv2LACgatWqsLOT32NIRERERPRUYR8H029VynH16lVcvXoVfn5+sLOzg1Il/EgRERERUamVc8XB2FKSmVw43Lx5E23btkX16tXRoUMHXL16FQAwaNCgYjuiEhERERHRY2EfB9MLhxEjRsDS0hIXL15EmTJl9Ot79uyJrVu3FmjjiIiIiIiKhSdQOMyYMQMajQbDhw/Xr0tPT0doaCjKli0Le3t79OjRA9euXTN43MWLFxESEoIyZcrAzc0No0aNQnZ29uM1Jg8m93HYvn07tm3bhkqVKhms9/Pzw4ULFwqsYURERERExUVhd44+dOgQPvnkE9SrV89g/YgRI/DTTz/hm2++gZOTE8LCwtC9e3f88ccfAACtVouQkBB4eHhg7969uHr1Kvr27QtLS0tMnz790RuUB5OvOKSlpRlcachx69YtWFtbF0ijiIiIiIiKFROuOKSkpBgsGRkZ+aZOTU1Fnz59sGLFCoNJlpOTk/HZZ59h7ty5eP7559GoUSOsXLkSe/fuxf79+wHc/1H/+PHj+Oqrr9CgQQO0b98eU6ZMweLFi5GZmVmgh8DkwqFly5b44osv9P/WaDTQ6XSYNWsW2rRpU6CNIyIiIiIqDjQ62QIAXl5ecHJy0i8RERH55g4NDUVISAiCgoIM1kdHRyMrK8tgfc2aNeHt7Y19+/YBAPbt24e6devC3f1/M7gHBwcjJSUFx47JZxeXMPlWpVmzZqFt27Y4fPgwMjMzMXr0aBw7dgy3bt3SXzIhIiIiIipRTBiONT4+Ho6OjvrV+d2Vs27dOsTExODQoUO5tiUkJMDKygrOzs4G693d3ZGQkKCPebBoyNmes60gmVw41KlTB6dOncLHH38MBwcHpKamonv37ggNDUWFChUKtHFERERERMWBKX0cHB0dDQqHh4mPj8e7776LHTt2wMbGpgBaWbhMLhx27dqFNm3a4IMPPsi1bfHixQgNDS2QhhERERERFRuFMAFcdHQ0EhMT8cwzz+jXabVaREVF4eOPP8a2bduQmZmJpKQkg6sO165dg4eHBwDAw8MDBw8eNMibM+pSTkxBMblw6N69O3bu3IlGjRoZrF+wYAHGjRtXbAuHM70dYCao5Kpsyr/zSo5bdeSvjLJ/Ooji4m+LU+KKViuK+/OstzinVZksUdyk3l3EOa91l+2/Z9M3xTnN4mUVucN5cUqcnO4qipv0T0dxTosXb4ji5i19SZyzwjnZ0GrNJ+8X54y57SWONfvUShT3TfwzxoP+ZT1Xduyhk7/n4v4pJ4rrufOAOOfvrzYUxen+/kecc+eXjYwHAXDrIR94wjJFI4o70ddPnHPnxkuiuIXze4hzukWniuLig2SfnwCQ5Sh7jVh0uCPOadszXRSXlmkpzjnj3b7i2Isv60RxNhmy8w4ATaPCRHE11pwX59wdWFUUVzFTPjykj5XsM7TWomRxTvPWsuPZ4+NR4pxmzWV/vD2s5V+5PjzWVRSXmW0uzhlRb5Mors/RAeKcq+qsFsWt7hFsNCZbmwEgRrzvIlEIhUPbtm1x9OhRg3UDBgxAzZo1MWbMGHh5ecHS0hKRkZHo0eP+Z+zJkydx8eJFBAYGAgACAwMxbdo0JCYmws3NDQCwY8cOODo6wt/f37QGGWFy4TB79my0b98eUVFRqFmzJgBgzpw5mDx5Mn766acCbRwRERERUXGg+XcxFmMKBwcH1KlTx2CdnZ0dypYtq18/aNAghIeHw9XVFY6Ojhg6dCgCAwPRtGlTAEC7du3g7++P119/HbNmzUJCQgI+/PBDhIaGFviIpyYXDm+88QZu3bqFoKAg7NmzB+vXr8f06dPx888/o3nz5gXaOCIiIiKiYqEQrjhIzJs3D2ZmZujRowcyMjIQHByMJUuW6Lebm5vjxx9/xNtvv43AwEDY2dmhX79+mDx5coG3xeTCAQBGjx6NmzdvonHjxtBqtdi2bZu+6jFFVFQUZs+ejejoaFy9ehWbNm1C165d9duVUpgwYQJWrFiBpKQkNG/eHEuXLoWfn/wSOxERERHR4yrsCeBy/Pbbbwb/trGxweLFi7F48eKHPsbHxwc///zz4+/cCFHhsHDhwlzrKlasiDJlyqBVq1Y4ePCgvlPGsGHDxDtPS0tD/fr1MXDgQHTv3j3X9lmzZmHhwoVYvXo1fH19MW7cOAQHB+P48eNPRc9zIiIiIiohiuiKQ3EiKhzmzZuX53pzc3P88ccf+vkbNBqNSYVD+/bt0b59+zy3KaUwf/58fPjhh+jS5X5n3C+++ALu7u7YvHkzevXqJd4PEREREdFjK+GFgTGiwiEuLq6w25HnPhMSEgxmynNyckJAQAD27dv30MIhIyPDYFrvlJSUQm8rEREREZVsD84MnV9MSWZW1A14mJyZ7vKaCS+/WfAiIiIMpvj28pIPNUlERERElJecPg7GlpLM5MKhR48emDlzZq71s2bNwssvv1wgjXocY8eORXJysn6Jj48v6iYRERER0dNOCZcSzOTCISoqCh06dMi1Pmduh4KSM9Ndzsx3OR6cKS8v1tbW+mm+pdN9ExERERHlh1ccHqFwSE1NhZVV7tljLS0tC7Q/ga+vLzw8PBAZGalfl5KSggMHDuhnyiMiIiIieiJ4xcH0wqFu3bpYv359rvXr1q0zeVrr1NRUxMbGIjY2FsD9DtGxsbG4ePEiNBoNhg8fjqlTp+L777/H0aNH0bdvX3h6ehrM9UBEREREVOhYOJg+Ady4cePQvXt3nD17Fs8//zwAIDIyEl9//TW++eYbk3IdPnwYbdq00f87PDwcANCvXz+sWrUKo0ePRlpaGoYMGYKkpCS0aNECW7du5RwORERERPREPakJ4IozkwuHTp06YfPmzZg+fTq+/fZb2Nraol69eti5cyeee+45k3K1bt0aSj38CGs0GkyePLlQpswmIiIiIhLjBHCmFw4AEBISgpCQkIJuCxERERFRsaRRCpp8fvDOiSnJHqlwKMk02bKZO6yS5d1Dwr/ZIIob+clgcc5N1RuI4trXOSbOuf/zhqK41IbyW8UynWRxZuZacU4X4VNK75Ekzml1yEWWU1dGnNPvxbOiuL9qOYtzLnp3uShu5vOdxTkv9K8kjnWvkSWKy9wg/2ixzcowHgTgtp+1OKdliqso7ss3O4lzWt29LYq7+ZOfOGegi2xyzRun5OfI5S/ZOQrdskWcc3EH2Q9F5V3TxDlvTpCd94oR5uKcpwfKXnd+794S57zSrbIortxf98Q553+5UBz79slXRXE2y+UjCJ6vI/tik+VTXpxTHbATxems5K+RxSN7iuLsky+Lc04+2FEUV+ubS+Kc2Cj7PnCxu6c4pZP1w+eretC9DNlxB4Cpp2Tv44xfy4lzvqYdIIoza278b6w2Mx04Lt51keAEcI9QOGi1WsybNw8bNmzAxYsXkZmZabD91i35BzIRERER0VOBtyqZPqrSpEmTMHfuXPTs2RPJyckIDw9H9+7dYWZmhokTJxZCE4mIiIiIihbncXiEwmHNmjVYsWIFRo4cCQsLC/Tu3Ruffvopxo8fj/379xdGG4mIiIiIihaHYzW9cEhISEDdunUBAPb29khOTgYAdOzYET/99FPBto6IiIiIqBjgFYdHKBwqVaqEq1evAgCqVq2K7du3AwAOHToEa2t550UiIiIioqcGrziYXjh069YNkZGRAIChQ4di3Lhx8PPzQ9++fTFw4MACbyARERERUXFQmq82AI8wqtKMGTP0/92zZ0/4+Phg79698PPzQ6dO8qENiYiIiIieGkrdX4zFlGAmFw5RUVFo1qwZLCzuP7Rp06Zo2rQpsrOzERUVhVatWhV4I4mIiIiIipLkqkJJv+pg8q1Kbdq0yXOuhuTkZLRp06ZAGkVEREREVJxotLKlJDP5ioNSChqNJtf6mzdvws5OPoMhEREREdFTgxPAyQuH7t27AwA0Gg369+9vMIKSVqvFkSNH0KxZs4JvIRERERFREeOtSiYUDk5OTgDuX3FwcHCAra2tfpuVlRWaNm2KwYMHF3wLiYiIiIiKGjtHywuHlStXAgAqV66M9957j7clEREREVGpwSsOj9DHYfTo0VAPVFMXLlzApk2b4O/vj3bt2hVo4wpS+cOAhaXxuDOv2Yjy1ZpxUbzvt1wHieKqHEoX5/wysIkozmuy/BVs7S+LtT+eKM6ZEFBBFNeg0mVxzkOBVUVxFb9yFOd03HlMFJcUXEucM2G5ryiuytUscc7pYY1FcacWyI47ANSccFIce+ETD1Gc0wYHcc6rgbL3nM1N+Wv5VH9b40EAXI6Yi3OmDHERxVXrKnstAcDByY1EcVVPHBHnVFpZz7xfbtcX55T+gnb6NdlxB4Cao+6J4s70leeEWaYozO97+WfYjQU+orizL1uJcw5+f4Q4Vnq/dFL13H0PH8byiCw2rov8PZftlC2Ku5FURpzzdoDsfNoNcBbn9FgvO08pz8g/Qy8Lx4Xx+/KOOOepKuVkgdY6cc5mVU+I4lz6yT/DVh1oLoqrFXXDaEy2NkO83yLDPg6mj6rUpUsXfPHFFwCApKQkNGnSBHPmzEGXLl2wdOnSAm8gEREREVFRMzb5W2mYBM7kwiEmJgYtW7YEAHz77bfw8PDAhQsX8MUXX2DhwoUF3kAiIiIioiKX08fB2FKCmVw43L17Fw4O929D2L59O7p37w4zMzM0bdoUFy5cMClXVFQUOnXqBE9PT2g0GmzevFm/LSsrC2PGjEHdunVhZ2cHT09P9O3bF1euXDG1yUREREREj4VXHB6hcKhWrRo2b96M+Ph4bNu2Td+vITExEY6O8nvKASAtLQ3169fH4sWLc227e/cuYmJiMG7cOMTExOC7777DyZMn0blzZ1ObTERERET0WDQ62VKSmdw5evz48Xj11VcxYsQItG3bFoGBgQDuX31o2LChSbnat2+P9u3b57nNyckJO3bsMFj38ccfo0mTJrh48SK8vb1NbToRERER0aPRqfuLsZgSzOTC4aWXXkKLFi1w9epV1K//v1E52rZti27duhVo4/4rOTkZGo0Gzs7OD43JyMhARsb/euanpKQUapuIiIiIqBTgqEqmFw4A4OHhAQ8Pw2EZmzSRDQ/6qNLT0zFmzBj07t0731uiIiIiMGnSpEJtCxERERGVLhoI5nF4Ii0pOib3cSgKWVlZeOWVV6CUMjrk69ixY5GcnKxf4uPjn1AriYiIiKjE4qhKxb9wyCkaLly4gB07dhjtgG1tbQ1HR0eDhYiIiIjocRTGqEoRERF49tln4eDgADc3N3Tt2hUnTxpOzJqeno7Q0FCULVsW9vb26NGjB65du2YQc/HiRYSEhKBMmTJwc3PDqFGjkJ0tm5jRFMW6cMgpGk6fPo2dO3eibNmyRd0kIiIiIiqNlHAxwe7duxEaGor9+/djx44dyMrKQrt27ZCWlqaPGTFiBH744Qd888032L17N65cuYLu3bvrt2u1WoSEhCAzMxN79+7F6tWrsWrVKowfP/4xn3Buj9THoaCkpqbizJkz+n/HxcUhNjYWrq6uqFChAl566SXExMTgxx9/hFarRUJCAgDA1dUVVlayaeOJiIiIiB6XRilojNyKZGz7f23dutXg36tWrYKbmxuio6PRqlUrJCcn47PPPsPatWvx/PPPAwBWrlyJWrVqYf/+/WjatCm2b9+O48ePY+fOnXB3d0eDBg0wZcoUjBkzBhMnTizQ78xFesXh8OHDaNiwoX4Y1/DwcDRs2BDjx4/H5cuX8f333+PSpUto0KABKlSooF/27t1blM0mIiIiotJGJ1xwf1TPB5cHR/zMT3JyMoD7P5IDQHR0NLKyshAUFKSPqVmzJry9vbFv3z4AwL59+1C3bl24u7vrY4KDg5GSkoJjx449xhPOrUivOLRu3Roqn8osv21ERERERE+KKVccvLy8DNZPmDABEydOzPexOp0Ow4cPR/PmzVGnTh0AQEJCAqysrHJNReDu7q6/EychIcGgaMjZnrOtIBVp4UBERERE9FQwYQK4+Ph4gwF6rK2tjaYPDQ3F33//jT179jxWMwtTqSkcmg4/DGt7S6Nxm/+pbzQGAGBCT3XXo7JRfTNcjLcvR6UpskteZjeSxTmvBTrI4pq6Gw/6l8de2VWjw67VxDk1Wlnc5Rfk875faVlLHCvlN+KQKM6sbnVxzvNfy9rpsMdcnPNWsJ84tpLzJVGcd/h5cc745rLXcmrXRuKcsJGde2Um/wg0u2Ajirs4urE4p7QXXeYW+cAQVh0TRXG/nPQX56x5T5bT9U/53a8ptV1FcWWPyK88J7jK9r/9O/m8Q069rxkPAmCRZC/P+dM5cazfrnRR3J5P5a+7u96yv1/mDlninEiW/f3qGbpTnHLre8+J4lIrykdPTGgmez35rbkrzllrjuzv7OkIF3FOz+9k3xuuPC9/z51sZfyLKwBoLO3EOTVTZPs/M76M0RjdXTNgoHjXRUIyalLOdlNH9gwLC8OPP/6IqKgoVKpUSb/ew8MDmZmZSEpKMrjqcO3aNf2cah4eHjh48KBBvpxRl/4779rjKtajKhERERERFQuFMI+DUgphYWHYtGkTfv31V/j6+hpsb9SoESwtLREZGalfd/LkSVy8eBGBgYEAgMDAQBw9ehSJif/7kSdnCgN/f/mPRBKl5ooDEREREdGj0ujuL8ZiTBEaGoq1a9diy5YtcHBw0PdJcHJygq2tLZycnDBo0CCEh4fD1dUVjo6OGDp0KAIDA9G0aVMAQLt27eDv74/XX38ds2bNQkJCAj788EOEhoaKbpEyBQsHIiIiIiJjJFcUTLzisHTpUgD3Bwx60MqVK9G/f38AwLx582BmZoYePXogIyMDwcHBWLJkiT7W3NwcP/74I95++20EBgbCzs4O/fr1w+TJk01qiwQLByIiIiIiYyQTvJk4IKhkBFEbGxssXrwYixcvfmiMj48Pfv75Z9N2/ghYOBARERERGVEYE8A9bVg4EBEREREZUwi3Kj1tWDgQERERERmjoJ8ZOt+YEoyFAxERERGRERqdgsbIsEkaYxPEPeVYOBARERERGcNblVg4EBEREREZpQNgbFJvE+dxeNqwcCAiIiIiMoKjKrFwICIiIiIyjrcqsXAgIiIiIjKKhUPpKRz+6VYWFmZWRuN084zdvHZf3BtVxPvu+dJvorjNy1uLc96q6SSK8/zdUpyz2tfporjLz9mJcyb7yuJsL4tTotKvqaI4naW5OGdCU9lx8tqcIM55YUyAKK5s66vinPXtk0Vxf7WoKM555578NeL6jo0obvi2neKcW2IaiOJWHZO9PgHA4wdbUVxStzvinH4jbonijk/wlOdcmSmKs1wiv2n2xisNRXH2h2SfdQCgbIx/dgKApttNcc7bf5QTxfl8lyjO6bJddo6S1ziLcybv8hDFlbknTonEV+uIY/+JzRLFeV/WinOmVpZ9Nn7U9itxzrEr+4vifnv9WXHO8+/KvoDVGn1GnNMuobJs3yPl748fmn4tigvr/Y445416ZqI4l4qy1zwAxH9ZWRRn+4OjOGe5KjdEceXfMf75na3LQJx4z0WEhUPpKRyIiIiIiB4ZO0ezcCAiIiIiMoadowHZtbBCEhUVhU6dOsHT0xMajQabN29+aOxbb70FjUaD+fPnP7H2EREREREBALQ62VKCFWnhkJaWhvr162Px4sX5xm3atAn79++Hp6f83mEiIiIiogKT08fB2FKCFemtSu3bt0f79u3zjbl8+TKGDh2Kbdu2ISQk5Am1jIiIiIjoQZLCgIVDkdHpdHj99dcxatQo1K5dW/SYjIwMZGRk6P+dkpJSWM0jIiIiotKCoyoV7a1KxsycORMWFhYYNmyY+DERERFwcnLSL15eXoXYQiIiIiIqFXRKtpRgxbZwiI6OxoIFC7Bq1SpoNPLxlMeOHYvk5GT9Eh8fX4itJCIiIqJSQelkSwlWbAuH33//HYmJifD29oaFhQUsLCxw4cIFjBw5EpUrV37o46ytreHo6GiwEBERERE9FnaOLr59HF5//XUEBQUZrAsODsbrr7+OAQMGFFGriIiIiKhU0ikY7fxcwm9VKtLCITU1FWfO/G+q+Li4OMTGxsLV1RXe3t4oW7asQbylpSU8PDxQo0aNJ91UIiIiIirN2Dm6aAuHw4cPo02bNvp/h4eHAwD69euHVatWFVGriIiIiIj+Q6cAGOnDwCsOhad169ZQJlRm58+fL7zGEBERERE9jE4H44VDye4cXWz7OBARERERFRu8Van0FA53Py0DCztro3Ga6wW/718/aCGKc8nMFOf8YOlKUVxow1fFOX0+ksXZXZW/KdIqyIbS1TW8I845dtAaUdysJq3FOb0SXUVxjquSxDl1V+xEcf2894lzbhjQThSn7SrbNwBUapggjsVN2YSKfaeHi1M6nc8SxVW5fk+cc/0P80RxL/V6W5zz/6J+EMV9MPRNcc7rDWTnqcLNNHHOOz6y99zagbJjBAADM4eL4mwtrolzWj8ne93VfjlOnLOH82FRXL+1YeKcWjfZr4f+jc+Lc977wEMcq+uQLoqzTbAU57RIdRDFjf6zhzinz07ZZ7gmVf4+trjmJIqbfvBHcc5XPx8h27fFXXHOritGieKsnxGnhFmm7O9s2Zm24pzp5axEcQ6xl8Q54yrJ5spKE3wV0WakA3PEuy4aLBxKT+FARERERPTIOKoSCwciIiIiImOU0kEZmeDN2PanHQsHIiIiIiJjlDJ+RYG3KhERERERlXJKcKsSCwciIiIiolJOpwM0Rm5F4q1KRERERESlHK84sHAgIiIiIjJGabVQGm3+MSr/7U87Fg5ERERERMboFKDhFQciIiIiIsqPUgCM9XEo2YWDWVE3gIiIiIiouFM6JVoexeLFi1G5cmXY2NggICAABw8eLODWFwwWDkRERERExiidbDHR+vXrER4ejgkTJiAmJgb169dHcHAwEhMTC+FJPB4WDkRERERERhTWFYe5c+di8ODBGDBgAPz9/bFs2TKUKVMGn3/+eSE8i8dT4vs4qH/vNcu+mymK191NF8VpMzTiNmRnZYviVLa8Sk27I+u1rxU+HwDIljUT2kz5y0Z6nKTHHZA/92yd7JwDALQZorCsNHlO6bG/lyo88ACys2U5denm8pxpsucOyI+pNtOU112WKM5MeI4AIOWO7L0kPZ4AkCbNmSXPqc0UvpZNeO7aDNn+U4XPB5CfT1NeS9JP0IxU2esDANLMZc9Jly4/RzpjHSH/ZcpngymvO+1d2THN1spHctFmWIriTPlclv79UCa8lqXnyaTXsvD9oREedwBQwpxaE/4kqUzZ686U11J2lvAzTFfwnzcawesjJ5cqxn0EslWG0SsK2bj/mZWSkmKw3traGtbW1rniMzMzER0djbFjx+rXmZmZISgoCPv27SuAVhcsjSrOZ6gAXLp0CV5eXkXdDCIiIiIyIj4+HpUqVSrqZhhIT0+Hr68vEhISRPH29vZITU01WDdhwgRMnDgxV+yVK1dQsWJF7N27F4GBgfr1o0ePxu7du3HgwIHHantBK/FXHDw9PREfHw8HBwdoNPd/40pJSYGXlxfi4+Ph6OhYxC2kvPAcFX88R8Ufz1Hxx3P0dOB5KnxKKdy5cweenp5F3ZRcbGxsEBcXh8xM2WUjpZT+O2eOvK42PI1KfOFgZmb20MrV0dGRHwDFHM9R8cdzVPzxHBV/PEdPB56nwuXk5FTUTXgoGxsb2NjYFHjecuXKwdzcHNeuXTNYf+3aNXh4eBT4/h4XO0cTERERERUBKysrNGrUCJGRkfp1Op0OkZGRBrcuFRcl/ooDEREREVFxFR4ejn79+qFx48Zo0qQJ5s+fj7S0NAwYMKCom5ZLqSwcrK2tMWHChBJzv1lJxHNU/PEcFX88R8Ufz9HTgeeJClPPnj1x/fp1jB8/HgkJCWjQoAG2bt0Kd3f3om5aLiV+VCUiIiIiInp87ONARERERERGsXAgIiIiIiKjWDgQEREREZFRLByIiIiIiMioUlk4LF68GJUrV4aNjQ0CAgJw8ODBom5SqRUVFYVOnTrB09MTGo0GmzdvNtiulML48eNRoUIF2NraIigoCKdPny6axpZSERERePbZZ+Hg4AA3Nzd07doVJ0+eNIhJT09HaGgoypYtC3t7e/To0SPXZDZUeJYuXYp69erpJ6cKDAzEL7/8ot/O81O8zJgxAxqNBsOHD9ev4zkqehMnToRGozFYatasqd/Oc0RUCguH9evXIzw8HBMmTEBMTAzq16+P4OBgJCYmFnXTSqW0tDTUr18fixcvznP7rFmzsHDhQixbtgwHDhyAnZ0dgoODkZ6e/oRbWnrt3r0boaGh2L9/P3bs2IGsrCy0a9cOaWlp+pgRI0bghx9+wDfffIPdu3fjypUr6N69exG2unSpVKkSZsyYgejoaBw+fBjPP/88unTpgmPHjgHg+SlODh06hE8++QT16tUzWM9zVDzUrl0bV69e1S979uzRb+M5IgKgSpkmTZqo0NBQ/b+1Wq3y9PRUERERRdgqUkopAGrTpk36f+t0OuXh4aFmz56tX5eUlKSsra3V119/XQQtJKWUSkxMVADU7t27lVL3z4mlpaX65ptv9DEnTpxQANS+ffuKqpmlnouLi/r00095foqRO3fuKD8/P7Vjxw713HPPqXfffVcpxfdQcTFhwgRVv379PLfxHBHdV6quOGRmZiI6OhpBQUH6dWZmZggKCsK+ffuKsGWUl7i4OCQkJBicLycnJwQEBPB8FaHk5GQAgKurKwAgOjoaWVlZBuepZs2a8Pb25nkqAlqtFuvWrUNaWhoCAwN5foqR0NBQhISEGJwLgO+h4uT06dPw9PRElSpV0KdPH1y8eBEAzxFRjlI1c/SNGzeg1WpzzcTn7u6Of/75p4haRQ+TkJAAAHmer5xt9GTpdDoMHz4czZs3R506dQDcP09WVlZwdnY2iOV5erKOHj2KwMBApKenw97eHps2bYK/vz9iY2N5foqBdevWISYmBocOHcq1je+h4iEgIACrVq1CjRo1cPXqVUyaNAktW7bE33//zXNE9K9SVTgQ0eMJDQ3F33//bXDfLxUPNWrUQGxsLJKTk/Htt9+iX79+2L17d1E3iwDEx8fj3XffxY4dO2BjY1PUzaGHaN++vf6/69Wrh4CAAPj4+GDDhg2wtbUtwpYRFR+l6lalcuXKwdzcPNcoCNeuXYOHh0cRtYoeJuec8HwVD2FhYfjxxx+xa9cuVKpUSb/ew8MDmZmZSEpKMojneXqyrKysUK1aNTRq1AgRERGoX78+FixYwPNTDERHRyMxMRHPPPMMLCwsYGFhgd27d2PhwoWwsLCAu7s7z1Ex5OzsjOrVq+PMmTN8HxH9q1QVDlZWVmjUqBEiIyP163Q6HSIjIxEYGFiELaO8+Pr6wsPDw+B8paSk4MCBAzxfT5BSCmFhYdi0aRN+/fVX+Pr6Gmxv1KgRLC0tDc7TyZMncfHiRZ6nIqTT6ZCRkcHzUwy0bdsWR48eRWxsrH5p3Lgx+vTpo/9vnqPiJzU1FWfPnkWFChX4PiL6V6m7VSk8PBz9+vVD48aN0aRJE8yfPx9paWkYMGBAUTetVEpNTcWZM2f0/46Li0NsbCxcXV3h7e2N4cOHY+rUqfDz84Ovry/GjRsHT09PdO3ategaXcqEhoZi7dq12LJlCxwcHPT38zo5OcHW1hZOTk4YNGgQwsPD4erqCkdHRwwdOhSBgYFo2rRpEbe+dBg7dizat28Pb29v3LlzB2vXrsVvv/2Gbdu28fwUAw4ODvo+QTns7OxQtmxZ/Xqeo6L33nvvoVOnTvDx8cGVK1cwYcIEmJubo3fv3nwfEeUo6mGdisKiRYuUt7e3srKyUk2aNFH79+8v6iaVWrt27VIAci39+vVTSt0fknXcuHHK3d1dWVtbq7Zt26qTJ08WbaNLmbzODwC1cuVKfcy9e/fUO++8o1xcXFSZMmVUt27d1NWrV4uu0aXMwIEDlY+Pj7KyslLly5dXbdu2Vdu3b9dv5/kpfh4cjlUpnqPioGfPnqpChQrKyspKVaxYUfXs2VOdOXNGv53niEgpjVJKFVHNQkRERERET4lS1ceBiIiIiIgeDQsHIiIiIiIyioUDEREREREZxcKBiIiIiIiMYuFARERERERGsXAgIiIiIiKjWDgQEREREZFRLByIiIiIiMgoFg5ERA9o3bo1hg8fXqRt+O2336DRaJCUlPRYeZYvXw4vLy+YmZlh/vz5BdI2IiIqvSyKugFERFTwUlJSEBYWhrlz56JHjx5wcnIq6iYREdFTjoUDEVEJdPHiRWRlZSEkJAQVKlTIMyYzMxNWVlZPuGVERPS04q1KRFRqpaWloW/fvrC3t0eFChUwZ86cXDEZGRl47733ULFiRdjZ2SEgIAC//fabQcwff/yB1q1bo0yZMnBxcUFwcDBu376tf/ywYcPg5uYGGxsbtGjRAocOHTJ4/M8//4zq1avD1tYWbdq0wfnz53O1Y8+ePWjZsiVsbW3h5eWFYcOGIS0tLc/ntWrVKtStWxcAUKVKFWg0Gpw/fx4TJ05EgwYN8Omnn8LX1xc2NjYAgK1bt6JFixZwdnZG2bJl0bFjR5w9e1af7/z589BoNNiwYYO+Dc8++yxOnTqFQ4cOoXHjxrC3t0f79u1x/fp1g7Z8+umnqFWrFmxsbFCzZk0sWbIk/5NCRETFFgsHIiq1Ro0ahd27d2PLli3Yvn07fvvtN8TExBjEhIWFYd++fVi3bh2OHDmCl19+GS+++CJOnz4NAIiNjUXbtm3h7++Pffv2Yc+ePejUqRO0Wi0AYPTo0di4cSNWr16NmJgYVKtWDcHBwbh16xYAID4+Ht27d0enTp0QGxuLN954A++//75BG86ePYsXX3wRPXr0wJEjR7B+/Xrs2bMHYWFheT6vnj17YufOnQCAgwcP4urVq/Dy8gIAnDlzBhs3bsR3332H2NhYAPcLqPDwcBw+fBiRkZEwMzNDt27doNPpDPJOmDABH374IWJiYmBhYYFXX30Vo0ePxoIFC/D777/jzJkzGD9+vD5+zZo1GD9+PKZNm4YTJ05g+vTpGDduHFavXv0op4uIiIqaIiIqhe7cuaOsrKzUhg0b9Otu3rypbG1t1bvvvquUUurChQvK3NxcXb582eCxbdu2VWPHjlVKKdW7d2/VvHnzPPeRmpqqLC0t1Zo1a/TrMjMzlaenp5o1a5ZSSqmxY8cqf39/g8eNGTNGAVC3b99WSik1aNAgNWTIEIOY33//XZmZmal79+7lue8///xTAVBxcXH6dRMmTFCWlpYqMTHxIUflvuvXrysA6ujRo0oppeLi4hQA9emnn+pjvv76awVARUZG6tdFRESoGjVq6P9dtWpVtXbtWoPcU6ZMUYGBgfnun4iIiif2cSCiUuns2bPIzMxEQECAfp2rqytq1Kih//fRo0eh1WpRvXp1g8dmZGSgbNmyAO5fcXj55Zcfuo+srCw0b95cv87S0hJNmjTBiRMnAAAnTpwwaAMABAYGGvz7r7/+wpEjR7BmzRr9OqUUdDod4uLiUKtWLfHz9vHxQfny5Q3WnT59GuPHj8eBAwdw48YN/ZWGixcvok6dOvq4evXq6f/b3d0dAPS3ROWsS0xMBHD/KsbZs2cxaNAgDB48WB+TnZ3NjtpERE8pFg5ERA+RmpoKc3NzREdHw9zc3GCbvb09AMDW1vaJtOPNN9/EsGHDcm3z9vY2KZednV2udZ06dYKPjw9WrFgBT09P6HQ61KlTB5mZmQZxlpaW+v/WaDR5rsspOlJTUwEAK1asyFUY/fdYEhHR04GFAxGVSlWrVoWlpSUOHDig//J9+/ZtnDp1Cs899xwAoGHDhtBqtUhMTETLli3zzFOvXj1ERkZi0qRJee7DysoKf/zxB3x8fAAAWVlZOHTokH6uiFq1auH77783eNz+/fsN/v3MM8/g+PHjqFat2mM957zcvHkTJ0+exIoVK/TPcc+ePY+d193dHZ6enjh37hz69Onz2PmIiKjosXAgolLJ3t4egwYNwqhRo1C2bFm4ubnhgw8+gJnZ/8aMqF69Ovr06YO+fftizpw5aNiwIa5fv47IyEjUq1cPISEhGDt2LOrWrYt33nkHb731FqysrLBr1y68/PLLKFeuHN5++22MGjUKrq6u8Pb2xqxZs3D37l0MGjQIAPDWW29hzpw5GDVqFN544w1ER0dj1apVBm0dM2YMmjZtirCwMLzxxhuws7PD8ePHsWPHDnz88cePdRxcXFxQtmxZLF++HBUqVMDFixdzdc5+VJMmTcKwYcPg5OSEF198ERkZGTh8+DBu376N8PDwAtkHERE9ORxViYhKrdmzZ6Nly5bo1KkTgoKC0KJFCzRq1MggZuXKlejbty9GjhyJGjVqoGvXrjh06JD+KkX16tWxfft2/PXXX2jSpAkCAwOxZcsWWFjc/11mxowZ6NGjB15//XU888wzOHPmDLZt2wYXFxcA92812rhxIzZv3oz69etj2bJlmD59ukEb6tWrh927d+PUqVNo2bIlGjZsiPHjx8PT0/Oxj4GZmRnWrVuH6Oho1KlTByNGjMDs2bMfOy8AvPHGG/j000+xcuVK1K1bF8899xxWrVoFX1/fAslPRERPlkYppYq6EUREREREVLzxigMRERERERnFwoGIiIiIiIxi4UBEREREREaxcCAiIiIiIqNYOBARERERkVEsHIiIiIiIyCgWDkREREREZBQLByIiIiIiMoqFAxERERERGcXCgYiIiIiIjGLhQERERERERv0/IR48nws0bV4AAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "import matplotlib.pylab as plt\n", "\n", @@ -311,15 +578,35 @@ "id": "a32b07d5", "metadata": {}, "source": [ - "## 5. Streaming text input (one subword per decode step)\n", - "\n", - "Same model, but push subword ids **one at a time** as it decodes (the live-client\n", - "path). `omni.generate(...)` is given an async generator of `StreamingInput` chunks:\n", - "chunk 0 is the prefill (speaker + context, no text), each later chunk carries one\n", - "`text_token` with `max_tokens=1` (one chunk -> one decoded frame). After the text\n", - "is exhausted we feed the `-1` mask sentinel until the model emits the audio-EOS\n", - "token. Input is paced by output via `go_queue`. Codes are collected exactly as in\n", - "§4 (keep the cumulative tensor, slice off prefix + warm-up + EOS)." + "## 5. Streaming text input (chunked subword feed)\n", + "\n", + "Same model, but push subword ids as a live caller would — **several at a time**, in\n", + "**one `StreamingInput` per chunk** (fewer ZMQ round-trips).\n", + "\n", + "The model is fundamentally **one text subword per decode frame**: every step adds\n", + "exactly one `text_token` embedding to the backbone, so it can't *consume* N tokens\n", + "in a single frame. But the caller doesn't need to send one `StreamingInput` per\n", + "frame: it sends a chunk of N ids in a single `StreamingInput` with `max_tokens=N`,\n", + "and the engine **free-runs N frames** off that one chunk (exactly like the tail\n", + "chunk free-runs the acoustic tail). The model appends the chunk's ids to a\n", + "persistent `text_tokens` buffer and consumes one per frame, indexed by\n", + "`decode_offset`.\n", + "\n", + "Two rules make this exact:\n", + "\n", + "* `text_token` is always a **`list[int]`** (a single id is just `[id]`; an empty\n", + " list `[]` adds nothing — used for the tail).\n", + "* each chunk's **`max_tokens` equals the number of ids it carries**, so the segment\n", + " stops exactly when the buffer is consumed and the next chunk's ids are appended\n", + " exactly once.\n", + "\n", + "`prompt_token_ids` stays `[0]` for every decode chunk — its length is **not** the\n", + "token count (a length > 1 would be read as a prefill). A `produce_text_chunks`\n", + "coroutine emits `TEXT_TOKENS_PER_CHUNK` ids per round into `token_q`; set it to\n", + "`1`, `3`, `5`, … — the audio is identical, only the number of round-trips changes.\n", + "After the text is exhausted we feed an empty chunk (`text_token=[]`) with a large\n", + "`max_tokens` until the model emits the audio-EOS token. Codes are collected exactly\n", + "as in §4 (keep the cumulative tensor, slice off prefix + warm-up + EOS)." ] }, { @@ -327,7 +614,28 @@ "execution_count": null, "id": "aa57a573", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "streamed text ids (17): [22177, 1044, 1593, 1395, 1261, 2688, 1307, 1278, 49649, 30019, 19133, 3403, 1317, 16181, 3244, 1046, 131073]\n", + "delivering 6 chunk(s) of <= 3 token(s): [[22177, 1044, 1593], [1395, 1261, 2688], [1307, 1278, 49649], [30019, 19133, 3403], [1317, 16181, 3244], [1046, 131073]]\n" + ] + }, + { + "ename": "NameError", + "evalue": "name 'stream_params' is not defined", + "output_type": "error", + "traceback": [ + "\u001b[31m---------------------------------------------------------------------------\u001b[39m", + "\u001b[31mNameError\u001b[39m Traceback (most recent call last)", + "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[7]\u001b[39m\u001b[32m, line 143\u001b[39m\n\u001b[32m 139\u001b[39m producer.cancel()\n\u001b[32m 140\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m codes\n\u001b[32m 141\u001b[39m \n\u001b[32m 142\u001b[39m \n\u001b[32m--> \u001b[39m\u001b[32m143\u001b[39m codes_stream = \u001b[38;5;28;01mawait\u001b[39;00m run_streaming_request()\n\u001b[32m 144\u001b[39m audio_codes_stream = codes_stream[prompt_len + SPEECH_DELAY : -\u001b[32m1\u001b[39m]\n\u001b[32m 145\u001b[39m print(f\"cumulative codes : {tuple(codes_stream.shape)}\")\n\u001b[32m 146\u001b[39m print(f\"audio_codes : {tuple(audio_codes_stream.shape)}\")\n", + "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[7]\u001b[39m\u001b[32m, line 139\u001b[39m, in \u001b[36mrun_streaming_request\u001b[39m\u001b[34m()\u001b[39m\n\u001b[32m 135\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m getattr(co, \u001b[33m\"stop_reason\"\u001b[39m, \u001b[38;5;28;01mNone\u001b[39;00m) == AUDIO_STOP_TOKEN_ID \u001b[38;5;28;01mor\u001b[39;00m steps >= MAX_STEPS:\n\u001b[32m 136\u001b[39m \u001b[38;5;28;01mbreak\u001b[39;00m\n\u001b[32m 137\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m go_queue.put(\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[32m 138\u001b[39m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m139\u001b[39m producer.cancel()\n\u001b[32m 140\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m codes\n", + "\u001b[31mNameError\u001b[39m: name 'stream_params' is not defined" + ] + } + ], "source": [ "import asyncio\n", "from collections.abc import AsyncGenerator\n", @@ -341,28 +649,49 @@ "text_ids = list(tokenizer.encode(TEXT, add_special_tokens=False)) + [TEXT_EOS_ID]\n", "print(f\"streamed text ids ({len(text_ids)}): {text_ids}\")\n", "\n", - "stream_params = SamplingParams(\n", - " temperature=0.0,\n", - " max_tokens=1,\n", - " detokenize=False,\n", - " ignore_eos=True,\n", - " stop_token_ids=[AUDIO_STOP_TOKEN_ID],\n", - " output_kind=RequestOutputKind.DELTA,\n", - ")\n", + "# ── Chunked delivery ──────────────────────────────────────────────────────────\n", + "# A live caller emits several tokens at once. We send each chunk in ONE\n", + "# StreamingInput with max_tokens == chunk length, so the engine free-runs that many\n", + "# frames off a single message (fewer ZMQ round-trips). The model consumes one id per\n", + "# frame internally. Set to 1, 3, 5, ... — the audio is identical; only the number of\n", + "# round-trips changes. Mirrors benchmark_streaming_service.py --tokens-per-chunk.\n", + "TEXT_TOKENS_PER_CHUNK = 3\n", + "PRODUCER_DELAY_S = 0.0 # sleep between chunks to mimic a slow upstream producer\n", + "text_chunks = [text_ids[i : i + TEXT_TOKENS_PER_CHUNK] for i in range(0, len(text_ids), TEXT_TOKENS_PER_CHUNK)]\n", + "print(f\"delivering {len(text_chunks)} chunk(s) of <= {TEXT_TOKENS_PER_CHUNK} token(s): {text_chunks}\")\n", + "\n", + "\n", + "def make_stream_params(max_tokens: int) -> SamplingParams:\n", + " \"\"\"Audio-decode params for a streaming segment that emits `max_tokens` frames.\"\"\"\n", + " return SamplingParams(\n", + " temperature=0.0,\n", + " max_tokens=max_tokens,\n", + " detokenize=False,\n", + " ignore_eos=True,\n", + " stop_token_ids=[AUDIO_STOP_TOKEN_ID],\n", + " output_kind=RequestOutputKind.DELTA,\n", + " )\n", + "\n", + "\n", + "prefill_params = make_stream_params(1) # default for omni.generate + the prefill segment\n", + "streaming_params = make_stream_params(TEXT_TOKENS_PER_CHUNK)\n", "TAIL_MAX_TOKENS = DECODE_STEPS - len(text_ids)\n", - "tail_params = SamplingParams(\n", - " temperature=0.0,\n", - " max_tokens=TAIL_MAX_TOKENS,\n", - " detokenize=False,\n", - " ignore_eos=True,\n", - " stop_token_ids=[AUDIO_STOP_TOKEN_ID],\n", - " output_kind=RequestOutputKind.DELTA,\n", - ")\n", + "tail_params = make_stream_params(TAIL_MAX_TOKENS)\n", "\n", "go_queue: asyncio.Queue[bool] = asyncio.Queue()\n", + "_STREAM_END = object() # sentinel: producer is done sending text\n", + "\n", + "\n", + "async def produce_text_chunks(token_q: asyncio.Queue) -> None:\n", + " \"\"\"Simulate an upstream producer emitting TEXT_TOKENS_PER_CHUNK ids per round.\"\"\"\n", + " for chunk in text_chunks:\n", + " if PRODUCER_DELAY_S:\n", + " await asyncio.sleep(PRODUCER_DELAY_S)\n", + " await token_q.put(chunk)\n", + " await token_q.put(_STREAM_END)\n", "\n", "\n", - "async def stream_text_inputs() -> AsyncGenerator[StreamingInput, None]:\n", + "async def stream_text_inputs(token_q: asyncio.Queue) -> AsyncGenerator[StreamingInput, None]:\n", " # Prefill: speaker + context, NO text (its absence selects streaming-text mode).\n", " # The first subword rides on the first decode chunk, not the prefill.\n", " prefill_info = {\n", @@ -373,19 +702,43 @@ " }\n", " yield StreamingInput(\n", " prompt={\"prompt_token_ids\": [0] * prompt_len, \"additional_information\": prefill_info},\n", - " sampling_params=stream_params,\n", + " sampling_params=prefill_params, # max_tokens=1 -> the prefill segment emits 1 frame\n", " )\n", "\n", - " for tok in text_ids:\n", - " await go_queue.get()\n", + " # One StreamingInput per chunk: `text_token` is the whole chunk (a list[int]) and\n", + " # `max_tokens == len(chunk)`, so the engine free-runs that many frames off the one\n", + " # message. We release the next chunk only after the previous segment's frames are\n", + " # done (one `go_queue` token per output frame), so the chunk's ids are appended to\n", + " # the model's buffer before the next chunk replaces `text_token`. `prev_frames`\n", + " # tracks how many frames the last-yielded segment will emit (prefill -> 1).\n", + " prev_frames = 1\n", + " ended = False\n", + " chunks: list[list[int]] = []\n", + " while True:\n", + " while not chunks and not ended:\n", + " item = await token_q.get()\n", + " if item is _STREAM_END:\n", + " ended = True\n", + " else:\n", + " chunks.append([int(t) for t in item])\n", + " if not chunks:\n", + " break\n", + " chunk = chunks.pop(0)\n", + " for _ in range(prev_frames):\n", + " await go_queue.get()\n", + " params = streaming_params if len(chunk) == TEXT_TOKENS_PER_CHUNK else make_stream_params(len(chunk))\n", " yield StreamingInput(\n", - " prompt={\"prompt_token_ids\": [0], \"additional_information\": {\"text_token\": int(tok)}},\n", - " sampling_params=stream_params,\n", + " prompt={\"prompt_token_ids\": [0], \"additional_information\": {\"text_token\": chunk}},\n", + " sampling_params=params,\n", " )\n", + " prev_frames = len(chunk)\n", "\n", - " await go_queue.get()\n", + " # Tail: an empty chunk (`text_token=[]`) masks the text channel; the large\n", + " # `max_tokens` lets the engine free-run the acoustic tail to audio-EOS.\n", + " for _ in range(prev_frames):\n", + " await go_queue.get()\n", " yield StreamingInput(\n", - " prompt={\"prompt_token_ids\": [0], \"additional_information\": {\"text_token\": -1}},\n", + " prompt={\"prompt_token_ids\": [0], \"additional_information\": {\"text_token\": []}},\n", " sampling_params=tail_params,\n", " )\n", " # Generator ends here: the engine now decodes the acoustic tail uninterrupted\n", @@ -405,19 +758,24 @@ " codes = None\n", " MAX_STEPS = 4 * DECODE_STEPS + 16\n", " steps = 0\n", - " async for out in omni.generate(\n", - " stream_text_inputs(), sampling_params_list=[stream_params], request_id=request_id\n", - " ):\n", - " steps += 1\n", - " co = out.outputs[0] if out.outputs else None\n", - " payload = (out.multimodal_output or {}).get(\"audio_codes\")\n", - " if isinstance(payload, list):\n", - " payload = torch.cat([t for t in payload if isinstance(t, torch.Tensor)], dim=0) if payload else None\n", - " if isinstance(payload, torch.Tensor) and (codes is None or payload.shape[0] >= codes.shape[0]):\n", - " codes = payload.detach().cpu().to(torch.long)\n", - " if getattr(co, \"stop_reason\", None) == AUDIO_STOP_TOKEN_ID or steps >= MAX_STEPS:\n", - " break\n", - " await go_queue.put(True)\n", + " token_q: asyncio.Queue = asyncio.Queue()\n", + " producer = asyncio.ensure_future(produce_text_chunks(token_q))\n", + " try:\n", + " async for out in omni.generate(\n", + " stream_text_inputs(token_q), sampling_params_list=[prefill_params], request_id=request_id\n", + " ):\n", + " steps += 1\n", + " co = out.outputs[0] if out.outputs else None\n", + " payload = (out.multimodal_output or {}).get(\"audio_codes\")\n", + " if isinstance(payload, list):\n", + " payload = torch.cat([t for t in payload if isinstance(t, torch.Tensor)], dim=0) if payload else None\n", + " if isinstance(payload, torch.Tensor) and (codes is None or payload.shape[0] >= codes.shape[0]):\n", + " codes = payload.detach().cpu().to(torch.long)\n", + " if getattr(co, \"stop_reason\", None) == AUDIO_STOP_TOKEN_ID or steps >= MAX_STEPS:\n", + " break\n", + " await go_queue.put(True)\n", + " finally:\n", + " producer.cancel()\n", " return codes\n", "\n", "\n", diff --git a/examples/tts/easymagpie_vllm_omni/model_repository/easymp/1/model.py b/examples/tts/easymagpie_vllm_omni/model_repository/easymp/1/model.py index 1ea04514e312..099a114897a6 100644 --- a/examples/tts/easymagpie_vllm_omni/model_repository/easymp/1/model.py +++ b/examples/tts/easymagpie_vllm_omni/model_repository/easymp/1/model.py @@ -22,13 +22,16 @@ * **whole-text** (default) — a request carries the full ``text``; we build a single prompt and run ``AsyncOmni.generate(prompt, ...)``. -* **streaming-text** — the client pushes subword ``text_token`` ids one (or a few) - at a time across several requests sharing a ``stream_id`` (``stream_start`` on the - first, ``stream_end`` on the last). We feed those tokens as ``StreamingInput`` - chunks (prefill, then one chunk per subword with ``max_tokens=1``, then a free- - running acoustic tail) into a single ``AsyncOmni.generate(, ...)`` call. - All audio for the stream is sent back on the ``stream_start`` request's response - sender; the follow-up requests just feed tokens and close with no output. +* **streaming-text** — the client pushes subword ``text_token`` ids across several + requests sharing a ``stream_id`` (``stream_start`` on the first, ``stream_end`` on + the last), with however many ids it wants per request. We forward each client + message verbatim as one ``StreamingInput`` chunk — ``text_token`` is the whole + ``list[int]`` and ``max_tokens == len(chunk)`` so the engine free-runs that many + frames off the single message (prefill first, then one chunk per client message, + then a free-running acoustic tail) into a single + ``AsyncOmni.generate(, ...)`` call. All audio for the stream is sent + back on the ``stream_start`` request's response sender; the follow-up requests + just feed tokens and close with no output. Both flavours converge on the same accumulator/codec pipeline: 1. Each engine step yields the *cumulative* ``audio_codes`` ``(T_total, C*S)`` @@ -221,6 +224,20 @@ def _init_sampling_helpers(self): self._RequestOutputKind = RequestOutputKind self._StreamingInput = StreamingInput + # Streaming chunks reuse SamplingParams keyed by max_tokens (== chunk len): + # one int per distinct chunk size the client sends, instead of cloning per + # chunk. Shared read-only across requests (the scheduler only reads them). + self._sp_cache: dict[int, object] = {} + + def _sampling_params(self, max_tokens: int): + """Return a cached :class:`SamplingParams` for ``max_tokens`` (>=1).""" + key = max(1, int(max_tokens)) + sp = self._sp_cache.get(key) + if sp is None: + sp = self._make_sampling_params(key) + self._sp_cache[key] = sp + return sp + def _make_sampling_params(self, max_tokens: int): """SamplingParams shared by both paths (audio sampling happens in the LT). @@ -596,28 +613,35 @@ async def _synthesize_whole_text(self, text: str, context_text: str, speaker: st ) await self._drive_codec(gen, response_sender, request_id, speaker, text, prompt_len) - def _text_chunk(self, text_token: int, sampling_params): + def _text_chunk(self, text_tokens, sampling_params): + """One ``StreamingInput`` carrying a whole chunk of ids as a ``list[int]``.""" return self._StreamingInput( - prompt={"prompt_token_ids": [0], "additional_information": {"text_token": int(text_token)}}, + prompt={ + "prompt_token_ids": [0], + "additional_information": {"text_token": [int(t) for t in text_tokens]}, + }, sampling_params=sampling_params, ) async def _stream_inputs(self, session: _StreamingSession): """Yield ``StreamingInput`` chunks for a streaming-text session. - Prefill (speaker + context, no text), then one ``max_tokens=1`` chunk per - subword id, then — once the client signals ``stream_end`` — the text-EOS id - and a ``-1`` mask sentinel whose raised ``max_tokens`` lets the model free-run - the acoustic tail to audio-EOS. - - Every post-prefill chunk is gated on ``session.pace_q`` (one decode-step - output == one chunk released) so the eagerly-drained feed can't overwrite a - not-yet-consumed ``text_token``; content tokens are additionally gated on the - client actually having sent them. + Prefill (speaker + context, no text) emits one frame, then we forward each + client message verbatim as one chunk: ``text_token`` is the whole + ``list[int]`` and ``max_tokens == len(chunk)`` so the engine free-runs that + many frames off the single message (the model appends the ids to its buffer + and consumes one per frame). Once the client signals ``stream_end`` we append + the text-EOS id and then a ``[]`` tail chunk whose raised ``max_tokens`` lets + the model free-run the acoustic tail to audio-EOS. + + Each post-prefill chunk is gated on ``session.pace_q`` — one decode-step + output per emitted frame, ``prev_frames`` of them — so the eagerly-drained + feed can't overwrite a chunk's ``text_token`` before the model has consumed + all of it. """ StreamingInput = self._StreamingInput prompt_len = self._prompt_len(session.speaker) - sp1 = self._make_sampling_params(1) + sp1 = self._sampling_params(1) prefill_info = { "speaker_id": session.speaker, @@ -625,45 +649,48 @@ async def _stream_inputs(self, session: _StreamingSession): "temperature": self.lt_temperature, "top_k": self.lt_top_k, } - # Prefill is released immediately; its decode-step output unblocks the first - # text token (mirrors the demo's go_queue handshake). + # Prefill is released immediately; its single decode-step output unblocks the + # first text chunk (mirrors the demo's go_queue handshake). yield StreamingInput( prompt={"prompt_token_ids": [0] * prompt_len, "additional_information": prefill_info}, sampling_params=sp1, ) + # Frames the last-yielded segment will emit (prefill -> 1); each gates the + # next chunk on that many pace_q tokens. + prev_frames = 1 n_text = 0 - pending: list = [] ended = False - while True: - # Refill from the client; block only while we have nothing buffered. - while not pending and not ended: - item = await session.token_q.get() - if item is _STREAM_END: - ended = True - else: - pending.extend(int(t) for t in item) - if not pending: + while not ended: + item = await session.token_q.get() + if item is _STREAM_END: + ended = True break + chunk = [int(t) for t in item] + if not chunk: + continue + for _ in range(prev_frames): + await session.pace_q.get() + yield self._text_chunk(chunk, self._sampling_params(len(chunk))) + prev_frames = len(chunk) + n_text += len(chunk) + + for _ in range(prev_frames): await session.pace_q.get() - tok = pending.pop(0) - yield self._text_chunk(tok, sp1) - n_text += 1 - - await session.pace_q.get() - yield self._text_chunk(self.text_eos_id, sp1) + yield self._text_chunk([self.text_eos_id], sp1) + prev_frames = 1 n_text += 1 - await session.pace_q.get() + for _ in range(prev_frames): + await session.pace_q.get() tail_budget = self.max_new_tokens - n_text - tail_params = self._make_sampling_params(tail_budget) - yield self._text_chunk(-1, tail_params) + yield self._text_chunk([], self._sampling_params(tail_budget)) async def _synthesize_streaming(self, session: _StreamingSession): prompt_len = self._prompt_len(session.speaker) inputs_gen = self._stream_inputs(session) gen = self.omni.generate( - inputs_gen, sampling_params_list=[self._make_sampling_params(1)], request_id=session.request_id + inputs_gen, sampling_params_list=[self._sampling_params(1)], request_id=session.request_id ) try: await self._drive_codec( From 1fba7d33b8f7b72114bf639ee93ba5d50c5c4ef1 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimkov Date: Tue, 16 Jun 2026 00:52:09 +0200 Subject: [PATCH 44/45] examples/tts/easymagpie_vllm_omni: update benchmarking with streaming chunks of tokens Signed-off-by: Viacheslav Klimkov --- .../easymagpie_vllm_omni/benchmark_model.py | 69 +++++++-- .../benchmark_streaming_service.py | 138 +++++++++++++----- 2 files changed, 155 insertions(+), 52 deletions(-) diff --git a/examples/tts/easymagpie_vllm_omni/benchmark_model.py b/examples/tts/easymagpie_vllm_omni/benchmark_model.py index 641192b52a56..706241144735 100644 --- a/examples/tts/easymagpie_vllm_omni/benchmark_model.py +++ b/examples/tts/easymagpie_vllm_omni/benchmark_model.py @@ -16,9 +16,10 @@ Two input modes, selectable with ``--streaming``: * whole-text (default) — the full target text is handed to the engine up front. -* streaming-text — subword ids are pushed one at a time as the model decodes - (prefill chunk, then one ``StreamingInput`` chunk per subword with - ``max_tokens=1``, then a free-running acoustic tail). +* streaming-text — subword ids are pushed as the model decodes, ``--tokens-per-chunk`` + ids at a time (prefill chunk, then one ``StreamingInput`` per chunk carrying a + ``list[int]`` of ids with ``max_tokens == len(chunk)`` so the engine free-runs + that many frames off one message, then a free-running acoustic tail). Both run on the same engine config. Reports throughput, TTFT, ITL (mean + p95), EOS hit rate and overall RTF. @@ -26,6 +27,7 @@ Usage: python benchmark_model.py --model ./easymp_vllm_model --num-requests 50 python benchmark_model.py --model ./easymp_vllm_model -n 50 --streaming + python benchmark_model.py --model ./easymp_vllm_model -n 50 --streaming --tokens-per-chunk 3 python benchmark_model.py --model ./easymp_vllm_model -n 50 -c 1 4 8 """ @@ -394,12 +396,19 @@ def _clone_sampling_params(sampling_params, max_tokens: int): return sp -def build_streaming_request(text: str, meta: ModelMeta, stream_params, max_new_tokens: int): +def build_streaming_request( + text: str, meta: ModelMeta, stream_params, max_new_tokens: int, tokens_per_chunk: int = 1 +): """Async ``StreamingInput`` feed + pacing coroutine (§5 of the demo). - Prefill (speaker + context, no text), one chunk per subword with - ``max_tokens=1``, then a ``-1`` mask sentinel with a larger tail budget so the - model free-runs to audio-EOS. Input is paced by output via the queue. + Prefill (speaker + context, no text), then one ``StreamingInput`` per chunk of + up to ``tokens_per_chunk`` subword ids: ``text_token`` is the whole chunk (a + ``list[int]``) and the chunk's ``max_tokens`` equals its length, so the engine + free-runs that many frames off one message (fewer round-trips; the model still + consumes one id per frame internally). Finally an empty chunk (``text_token=[]``) + with a larger tail budget lets the model free-run to audio-EOS. Input is paced by + output via the queue: the next chunk is released only after the previous segment's + frames have all been emitted (one ``go_queue`` token per output frame). """ try: from vllm.engine.protocol import StreamingInput @@ -413,23 +422,38 @@ def build_streaming_request(text: str, meta: ModelMeta, stream_params, max_new_t } prefill_info.update(_speaker_info(meta)) text_ids = list(meta.tokenizer.encode(text, add_special_tokens=False)) + [meta.text_eos_id] + n = max(1, int(tokens_per_chunk)) + chunks = [text_ids[i : i + n] for i in range(0, len(text_ids), n)] + # Reuse a few fixed sampling_params instead of cloning per chunk: full-size + # chunks share ``chunk_params``; only a trailing partial chunk needs a one-off + # clone. ``stream_params`` (max_tokens=1) serves the prefill. + chunk_params = _clone_sampling_params(stream_params, n) tail_params = _clone_sampling_params(stream_params, max_new_tokens - len(text_ids)) go_queue: asyncio.Queue = asyncio.Queue() async def inputs(): + # Prefill emits one frame (stream_params.max_tokens == 1). yield StreamingInput( prompt={"prompt_token_ids": [0] * meta.prompt_len, "additional_information": prefill_info}, sampling_params=stream_params, ) - for tok in text_ids: - await go_queue.get() + prev_frames = 1 + for chunk in chunks: + for _ in range(prev_frames): + await go_queue.get() + params = chunk_params if len(chunk) == n else _clone_sampling_params(stream_params, len(chunk)) yield StreamingInput( - prompt={"prompt_token_ids": [0], "additional_information": {"text_token": int(tok)}}, - sampling_params=stream_params, + prompt={ + "prompt_token_ids": [0], + "additional_information": {"text_token": [int(t) for t in chunk]}, + }, + sampling_params=params, ) - await go_queue.get() + prev_frames = len(chunk) + for _ in range(prev_frames): + await go_queue.get() yield StreamingInput( - prompt={"prompt_token_ids": [0], "additional_information": {"text_token": -1}}, + prompt={"prompt_token_ids": [0], "additional_information": {"text_token": []}}, sampling_params=tail_params, ) @@ -453,6 +477,7 @@ async def worker( stream_params, streaming: bool, max_new_tokens: int, + tokens_per_chunk: int, results: list, counter: dict, lock: asyncio.Lock, @@ -469,7 +494,9 @@ async def worker( request_id = f"bench-easymp-w{worker_id}-{uuid.uuid4().hex[:8]}" if streaming: - inputs, pace = build_streaming_request(text, meta, stream_params, max_new_tokens) + inputs, pace = build_streaming_request( + text, meta, stream_params, max_new_tokens, tokens_per_chunk + ) result = await run_one_request( omni, inputs, stream_params, request_id, meta, pace=pace, max_steps=4 * max_new_tokens + 16 ) @@ -544,7 +571,7 @@ async def _run_workers(omni, texts, meta, sampling_params, stream_params, args, asyncio.create_task( worker( i, omni, texts, meta, sampling_params, stream_params, - args.streaming, args.max_new_tokens, results, counter, lock, + args.streaming, args.max_new_tokens, args.tokens_per_chunk, results, counter, lock, ) ) for i in range(concurrency) @@ -589,7 +616,10 @@ async def main(args): ) if meta.prompt_len + args.max_new_tokens > args.max_model_len: logger.warning("prompt_len + max_new_tokens exceeds max_model_len (%d)", args.max_model_len) - logger.info("Mode: %s", "streaming-text" if args.streaming else "whole-text") + if args.streaming: + logger.info("Mode: streaming-text tokens_per_chunk=%d", max(1, args.tokens_per_chunk)) + else: + logger.info("Mode: whole-text") stage_cfg = _build_stage_config( max_num_seqs=max(args.concurrency), @@ -695,6 +725,13 @@ def parse_args(): parser.add_argument("--model", type=str, default="./easymp_vllm_model", help="Converted EasyMagpie model dir") parser.add_argument("--text-file", type=str, default=None, help="One utterance per line (optionally tab-sep)") parser.add_argument("--streaming", action="store_true", help="Benchmark the token-streamed input path") + parser.add_argument( + "--tokens-per-chunk", + type=int, + default=1, + help="Streaming only: number of subword ids to feed per StreamingInput chunk " + "(the engine free-runs that many frames off one message). Default: %(default)s.", + ) parser.add_argument("-c", "--concurrency", type=int, nargs="+", default=[1], help="Concurrency levels to test") parser.add_argument("-n", "--num-requests", type=int, default=50, help="Requests per concurrency level") parser.add_argument("--num-warmups", type=int, default=3, help="Warmup rounds (total = concurrency * this)") diff --git a/examples/tts/easymagpie_vllm_omni/benchmark_streaming_service.py b/examples/tts/easymagpie_vllm_omni/benchmark_streaming_service.py index 2f65c40881b3..6e1e410ecf6a 100644 --- a/examples/tts/easymagpie_vllm_omni/benchmark_streaming_service.py +++ b/examples/tts/easymagpie_vllm_omni/benchmark_streaming_service.py @@ -48,12 +48,13 @@ class RequestResult: uttid: str num_samples: int = 0 - elapsed_s: float = 0.0 + duration_s: float = 0.0 ttfa_s: float = 0.0 - reached_eos: bool = False - keepup_ratios: list[float] = field(default_factory=list) - audio: np.ndarray | None = None error: str | None = None + # Arrival time (relative to request start) and playback duration of each audio chunk. + chunk_arrivals: list[float] = field(default_factory=list) + chunk_durations: list[float] = field(default_factory=list) + audio: np.ndarray | None = None @dataclass @@ -146,8 +147,8 @@ def _send(inputs, request_id): num_samples = 0 t_first: float | None = None - t_prev: float | None = None - keepup: list[float] = [] + chunk_arrivals: list[float] = [] + chunk_durations: list[float] = [] audio_chunks: list[np.ndarray] = [] reached_eos = False @@ -159,11 +160,13 @@ def _send(inputs, request_id): result, error = result_q.get(timeout=chunk_timeout) except queue.Empty: elapsed = time.perf_counter() - t0 - return RequestResult(uttid=uttid, elapsed_s=elapsed, ttfa_s=elapsed, error="no chunk within chunk_timeout") + return RequestResult( + uttid=uttid, duration_s=elapsed, ttfa_s=elapsed, error="no chunk within chunk_timeout" + ) if error: elapsed = time.perf_counter() - t0 - return RequestResult(uttid=uttid, elapsed_s=elapsed, ttfa_s=elapsed, error=str(error)) + return RequestResult(uttid=uttid, duration_s=elapsed, ttfa_s=elapsed, error=str(error)) response = result.get_response() if response.id != start_rid: @@ -176,9 +179,8 @@ def _send(inputs, request_id): now = time.perf_counter() if t_first is None: t_first = now - else: - keepup.append((now - t_prev) / (audio.size / SAMPLE_RATE)) - t_prev = now + chunk_arrivals.append(now - t0) + chunk_durations.append(audio.size / SAMPLE_RATE) num_samples += int(audio.size) if save_audio: audio_chunks.append(np.asarray(audio, dtype=np.float32).reshape(-1)) @@ -201,10 +203,11 @@ def _send(inputs, request_id): return RequestResult( uttid=uttid, num_samples=num_samples, - elapsed_s=elapsed, + duration_s=elapsed, ttfa_s=ttfa, - reached_eos=reached_eos, - keepup_ratios=keepup, + error=None if reached_eos else "stream ended without final response", + chunk_arrivals=chunk_arrivals, + chunk_durations=chunk_durations, audio=(np.concatenate(audio_chunks) if save_audio and audio_chunks else None), ) @@ -263,7 +266,7 @@ def worker( if verbose: print( f"[worker {worker_id:02d}] req {task_idx} ({uttid}) — " - f"{result.num_samples / SAMPLE_RATE:.2f}s audio in {result.elapsed_s:.2f}s " + f"{result.num_samples / SAMPLE_RATE:.2f}s audio in {result.duration_s:.2f}s " f"(TTFA {result.ttfa_s * 1000:.0f}ms)" ) finally: @@ -342,34 +345,98 @@ def _percentile(sorted_vals: list[float], pct: float) -> float: return sorted_vals[idx] +def _mean(vals: list[float]) -> float: + return (sum(vals) / len(vals)) if vals else 0.0 + + +def _playback_metrics(arrivals: list[float], durations: list[float]) -> dict: + """Simulate gapless playback and detect buffer underruns. + + Playback starts when the first chunk arrives. An underrun occurs whenever a + chunk has not arrived yet by the time we finish playing everything buffered + so far (i.e. the player would stall waiting for it). Also reports the + inter-chunk gaps and the per-chunk realtime factor (playback time of a chunk + divided by the time it took to fetch it; >1 means we receive faster than we + play, so the stream is sustainable). + """ + n = len(arrivals) + if n == 0: + return {"chunks": 0, "underruns": 0, "gaps": [], "chunk_rtfs": []} + + underruns = 0 + gaps: list[float] = [] + chunk_rtfs: list[float] = [] + playback_end = arrivals[0] + durations[0] + for i in range(1, n): + gap = arrivals[i] - arrivals[i - 1] + gaps.append(gap) + if gap > 0: + chunk_rtfs.append(durations[i] / gap) + if arrivals[i] > playback_end: + underruns += 1 + playback_end = arrivals[i] + playback_end += durations[i] + return {"chunks": n, "underruns": underruns, "gaps": gaps, "chunk_rtfs": chunk_rtfs} + + def _summarize(stats: BenchmarkStats, wall_s: float, concurrency: int) -> dict: - eos = [r for r in stats.results if r.reached_eos] - audio_s = sum(r.num_samples for r in eos) / SAMPLE_RATE - ttfas_ms = sorted(r.ttfa_s * 1000 for r in eos) - keepup = sorted(ratio for r in eos for ratio in r.keepup_ratios) + successes = [r for r in stats.results if r.error is None] + failures = [r for r in stats.results if r.error is not None] + audio_s = sum(r.num_samples for r in successes) / SAMPLE_RATE + ttfts_ms = sorted(r.ttfa_s * 1000 for r in successes) + + itl_ms: list[float] = [] + chunk_rtfs: list[float] = [] + total_chunks = 0 + total_underruns = 0 + reqs_with_underrun = 0 + for r in successes: + pm = _playback_metrics(r.chunk_arrivals, r.chunk_durations) + total_chunks += pm["chunks"] + total_underruns += pm["underruns"] + if pm["underruns"] > 0: + reqs_with_underrun += 1 + itl_ms.extend(g * 1000 for g in pm["gaps"]) + chunk_rtfs.extend(pm["chunk_rtfs"]) + itl_ms.sort() + return { "concurrency": concurrency, - "failed": len(stats.results) - len(eos), + "ok": len(successes), + "failed": len(failures), "wall_s": wall_s, "audio_s": audio_s, - "rtx": audio_s / wall_s if wall_s > 0 else 0.0, - "tput": len(eos) / wall_s if wall_s > 0 else 0.0, - "ttfa_mean_ms": (sum(ttfas_ms) / len(ttfas_ms)) if ttfas_ms else 0.0, - "ttfa_p95_ms": _percentile(ttfas_ms, 0.95), - "keepup_mean": (sum(keepup) / len(keepup)) if keepup else 0.0, - "keepup_p95": _percentile(keepup, 0.95), + "rtf": audio_s / wall_s if wall_s > 0 else 0.0, + "tput": len(successes) / wall_s if wall_s > 0 else 0.0, + "ttft_mean_ms": _mean(ttfts_ms), + "ttft_p95_ms": _percentile(ttfts_ms, 0.95), + "itl_mean_ms": _mean(itl_ms), + "itl_p95_ms": _percentile(itl_ms, 0.95), + "total_chunks": total_chunks, + "total_underruns": total_underruns, + "reqs_with_underrun": reqs_with_underrun, + "underrun_pct": (100.0 * total_underruns / total_chunks) if total_chunks else 0.0, + "playback_rtf_mean": _mean(chunk_rtfs), } def _print_summary(s: dict): print( - f"[concurrency={s['concurrency']}] rtx = synt / wall = " - f"{s['rtx']:.2f}x = {s['audio_s']:.2f} / {s['wall_s']:.0f}" + f"concurrency={s['concurrency']}: req/s {s['tput']:.2f}, " + f"ttft {s['ttft_mean_ms']:.1f}ms, itl {s['itl_mean_ms']:.1f}ms, " + f"rtf {s['rtf']:.2f}x, underrun {s['underrun_pct']:.1f}%" ) + + +def _print_detailed(s: dict): + print(f"[concurrency={s['concurrency']}] {s['ok']} ok / {s['failed']} failed") + print(f" req/s {s['tput']:.2f} | rtf {s['rtf']:.2f}x (audio {s['audio_s']:.0f}s / wall {s['wall_s']:.2f}s)") + print(f" ttft mean {s['ttft_mean_ms']:.1f}ms p95 {s['ttft_p95_ms']:.1f}ms") + print(f" itl mean {s['itl_mean_ms']:.1f}ms p95 {s['itl_p95_ms']:.1f}ms") print( - f"throughput={s['tput']:.2f} req/s; failed = {s['failed']}; " - f"TTFA={s['ttfa_mean_ms']:.1f} / {s['ttfa_p95_ms']:.1f} (p95); " - f"keepup={s['keepup_mean']:.3f} / {s['keepup_p95']:.3f} (p95)" + f" playback underruns {s['total_underruns']}/{s['total_chunks']} chunks " + f"({s['underrun_pct']:.2f}%) in {s['reqs_with_underrun']}/{s['ok']} reqs | " + f"realtime factor mean {s['playback_rtf_mean']:.2f}x (chunk play / fetch)" ) @@ -444,12 +511,11 @@ def main(): ) summary = _summarize(stats, wall_elapsed, concurrency) summaries.append(summary) - _print_summary(summary) + _print_detailed(summary) - if len(summaries) > 1: - print("\n=== Summary ===") - for s in summaries: - _print_summary(s) + print("\n=== Summary ===") + for s in summaries: + _print_summary(s) if __name__ == "__main__": From 39283e7b864aa324152ca4af328eea859f47fa6f Mon Sep 17 00:00:00 2001 From: Viacheslav Klimkov Date: Tue, 16 Jun 2026 12:51:46 +0200 Subject: [PATCH 45/45] examples/tts/easymagpie_vllm_omni: add scripts subdir Signed-off-by: Viacheslav Klimkov --- examples/tts/easymagpie_vllm_omni/README.md | 10 +- ...easy_magpietts_extract_speaker_encoding.py | 164 ----- .../easy_magpietts_single_infer.py | 141 ----- .../{ => scripts}/benchmark_model.py | 0 .../{ => scripts}/benchmark_service.py | 0 .../benchmark_streaming_service.py | 0 .../easy_magpietts_convert_to_vllm.py | 2 +- .../easymagpie_inference_demo.ipynb | 0 .../export_codec_decoder_onnx.py | 2 +- .../{ => scripts}/export_codec_decoder_trt.py | 2 +- .../{ => scripts}/run_service_request.ipynb | 0 .../scripts/streaming_codec_decode_demo.ipynb | 599 ++++++++++++++++++ 12 files changed, 607 insertions(+), 313 deletions(-) delete mode 100644 examples/tts/easymagpie_vllm_omni/easy_magpietts_extract_speaker_encoding.py delete mode 100644 examples/tts/easymagpie_vllm_omni/easy_magpietts_single_infer.py rename examples/tts/easymagpie_vllm_omni/{ => scripts}/benchmark_model.py (100%) rename examples/tts/easymagpie_vllm_omni/{ => scripts}/benchmark_service.py (100%) rename examples/tts/easymagpie_vllm_omni/{ => scripts}/benchmark_streaming_service.py (100%) rename examples/tts/easymagpie_vllm_omni/{ => scripts}/easy_magpietts_convert_to_vllm.py (99%) rename examples/tts/easymagpie_vllm_omni/{ => scripts}/easymagpie_inference_demo.ipynb (100%) rename examples/tts/easymagpie_vllm_omni/{ => scripts}/export_codec_decoder_onnx.py (99%) rename examples/tts/easymagpie_vllm_omni/{ => scripts}/export_codec_decoder_trt.py (97%) rename examples/tts/easymagpie_vllm_omni/{ => scripts}/run_service_request.ipynb (100%) create mode 100644 examples/tts/easymagpie_vllm_omni/scripts/streaming_codec_decode_demo.ipynb diff --git a/examples/tts/easymagpie_vllm_omni/README.md b/examples/tts/easymagpie_vllm_omni/README.md index 2aa447e21063..28fd6a7c1c8f 100644 --- a/examples/tts/easymagpie_vllm_omni/README.md +++ b/examples/tts/easymagpie_vllm_omni/README.md @@ -19,7 +19,7 @@ streaming requests. text tokenizer, and optional reference speaker embeddings. ```bash - python examples/tts/easymagpie_vllm_omni/easy_magpietts_convert_to_vllm.py \ + python examples/tts/easymagpie_vllm_omni/scripts/easy_magpietts_convert_to_vllm.py \ --nemo_file /2605_EMTTS_SmallMamba_Step150k_posttrained_epoch12.nemo \ --codec_model_path /25fps_spectral_codec_with_bandwidth_extension.nemo \ --outdir examples/tts/easymagpie_vllm_omni/easymp_vllm_model \ @@ -34,7 +34,7 @@ streaming requests. waveform (clamp specials → unstack → FSQ index-convert → decode baked in). ```bash - python examples/tts/easymagpie_vllm_omni/export_codec_decoder_onnx.py \ + python examples/tts/easymagpie_vllm_omni/scripts/export_codec_decoder_onnx.py \ --codec_model_path /25fps_spectral_codec_with_bandwidth_extension.nemo \ --nemo_file /2605_EMTTS_SmallMamba_Step150k_posttrained_epoch12.nemo \ --onnx-path examples/tts/easymagpie_vllm_omni/codec.onnx \ @@ -60,7 +60,7 @@ streaming requests. it into the Triton repo as `model.plan`. For now fp32 seems to be mandatory. ```bash - python examples/tts/easymagpie_vllm_omni/export_codec_decoder_trt.py \ + python examples/tts/easymagpie_vllm_omni/scripts/export_codec_decoder_trt.py \ --onnx-path examples/tts/easymagpie_vllm_omni/codec.onnx \ --trt-path examples/tts/easymagpie_vllm_omni/model_repository/codec/1/model.plan \ --batch-profile 1 8 32 --frames-profile 15 15 15 --fp32 @@ -75,5 +75,5 @@ streaming requests. ``` 7. **Send a request.** End-to-end gRPC streaming example in - [`run_server_request.ipynb`](run_service_request.ipynb) — sends `text`, - receives streamed `audio` chunks at 22.05 kHz. + [`scripts/run_service_request.ipynb`](scripts/run_service_request.ipynb) — + sends `text`, receives streamed `audio` chunks at 22.05 kHz. diff --git a/examples/tts/easymagpie_vllm_omni/easy_magpietts_extract_speaker_encoding.py b/examples/tts/easymagpie_vllm_omni/easy_magpietts_extract_speaker_encoding.py deleted file mode 100644 index 90f47dbd6a49..000000000000 --- a/examples/tts/easymagpie_vllm_omni/easy_magpietts_extract_speaker_encoding.py +++ /dev/null @@ -1,164 +0,0 @@ -# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -Standalone speaker-encoder output extractor for EasyMagpieTTS. - -Pre-computes ONLY the speaker-encoded context-audio embedding so it can be fed -to a separate (e.g. vLLM) backbone implementation. Context-text / task -embeddings are intentionally NOT included here -- the caller is expected to -prepend/append those (e.g. inside the vLLM model's ``preprocess``). - -This reproduces the audio branch of -``EasyMagpieTTSInferenceModel.prepare_context_tensors``:: - - audio -> codec codes -> (codec convert) -> add BOS/EOS -> frame stacking - -> per-codebook embedding -> speaker encoder - -and saves the resulting ``(T_audio, embedding_dim)`` tensor to disk. - -Example: - python examples/tts/easy_magpietts_extract_speaker_encoding.py \\ - --nemo_file /path/to/EMTTS_Pretraining_Qwen_WithCrossLingual_3_5_Delay.nemo \\ - --codec_model_path /path/to/25fps_spectral_codec_with_bandwidth_extension.nemo \\ - --phoneme_tokenizer_path /path/to/bpe_ipa_tokenizer_2048_en_de_es_fr_hi_it_vi_zh.json \\ - --context_audio /path/to/reference_voice.wav \\ - --out_file ./speaker_encoding.pt -""" -from __future__ import annotations - -import argparse - -import torch - -from nemo.collections.tts.modules.magpietts_inference.utils import ModelLoadConfig, load_easy_magpie_model -from nemo.collections.tts.modules.magpietts_modules import add_special_tokens -from nemo.utils import logging - - -def main(): - parser = argparse.ArgumentParser(description="Extract EasyMagpieTTS speaker-encoder output") - parser.add_argument("--nemo_file", required=True, help="Path to the EasyMagpieTTS .nemo checkpoint") - parser.add_argument("--codec_model_path", required=True, help="Path to the audio codec .nemo checkpoint") - parser.add_argument( - "--phoneme_tokenizer_path", - default=None, - help="Override the phoneme (IPA BPE) tokenizer path baked into the checkpoint. " - "Required if the path stored in the .nemo does not exist locally.", - ) - parser.add_argument("--context_audio", required=True, help="Reference/context wav for voice cloning") - parser.add_argument( - "--disable_cas_for_context_text", - action="store_true", - help="Set for legacy checkpoints trained without CAS embeddings on context text", - ) - parser.add_argument("--context_audio_duration", type=float, default=5.0) - parser.add_argument("--device", default="cuda") - parser.add_argument( - "--out_file", - default="./speaker_encoding.pt", - help="Output path. A torch .pt file (dict) is written; if it ends with .npy the " - "speaker-encoding tensor is saved as a NumPy array instead.", - ) - - args = parser.parse_args() - - model, ckpt_name = load_easy_magpie_model( - ModelLoadConfig( - nemo_file=args.nemo_file, - codecmodel_path=args.codec_model_path, - phoneme_tokenizer_path=args.phoneme_tokenizer_path, - disable_cas_for_context_text=args.disable_cas_for_context_text, - ), - device=args.device, - ) - logging.info(f"Loaded EasyMagpieTTS checkpoint: {ckpt_name}") - logging.info(f"use_speaker_encoder={getattr(model, 'use_speaker_encoder', False)}") - - device = next(model.parameters()).device - - with torch.inference_mode(): - # Load + trim context audio exactly like EasyMagpieTTSInferenceModel.do_tts. - context_audio = model._load_audio_for_inference(args.context_audio, model.sample_rate) - context_audio = model._adjust_audio_to_duration_for_inference( - context_audio, - model.sample_rate, - args.context_audio_duration, - model.codec_model_samples_per_frame, - ) - context_audio = context_audio.to(device) - context_audio_lens = torch.tensor([context_audio.size(1)], dtype=torch.long, device=device) - context_audio_codes, context_audio_codes_lens = model._codec_helper.audio_to_codes( - context_audio, context_audio_lens - ) - - # --- Audio branch of prepare_context_tensors (no context text / task embedding) --- - if model._codec_converter is not None: - context_audio_codes = model._codec_converter.convert_original_to_new( - audio_tokens=context_audio_codes, audio_lens=context_audio_codes_lens - ).long() - - context_audio_codes, context_audio_codes_lens = add_special_tokens( - codes=context_audio_codes, - codes_len=context_audio_codes_lens, - bos_id=model.context_audio_bos_id, - eos_id=model.context_audio_eos_id, - ) - - context_audio_codes, context_audio_codes_lens = model.stack_codes( - context_audio_codes, - context_audio_codes_lens, - model.context_audio_bos_id, - model.context_audio_eos_id, - model.frame_stacking_factor, - model.num_audio_codebooks, - ) - - context_audio_embedded = model.embed_audio_tokens(context_audio_codes) # (B, T_audio, E) - - if getattr(model, "use_speaker_encoder", False): - context_audio_embedded = model.encode_context_audio_embeddings( - context_audio_embedded=context_audio_embedded, - context_audio_lens=context_audio_codes_lens, - ) - else: - logging.warning( - "Checkpoint has use_speaker_encoder=False; saving raw per-codebook audio embeddings " - "(no speaker encoder applied)." - ) - - # Strip batch dim (B == 1) -> (T_audio, embedding_dim). - audio_len = int(context_audio_codes_lens[0].item()) - speaker_encoding = context_audio_embedded[0, :audio_len].contiguous().float().detach().cpu() - logging.info(f"Extracted speaker-encoder output: {tuple(speaker_encoding.shape)}") - - if args.out_file.endswith(".npy"): - import numpy as np - - np.save(args.out_file, speaker_encoding.numpy()) - else: - torch.save( - { - "speaker_encoding": speaker_encoding, - "context_audio": args.context_audio, - "embedding_dim": int(speaker_encoding.size(-1)), - "num_frames": int(speaker_encoding.size(0)), - "checkpoint": ckpt_name, - }, - args.out_file, - ) - logging.info(f"Wrote speaker encoding of shape {tuple(speaker_encoding.shape)} to {args.out_file}") - - -if __name__ == "__main__": - main() diff --git a/examples/tts/easymagpie_vllm_omni/easy_magpietts_single_infer.py b/examples/tts/easymagpie_vllm_omni/easy_magpietts_single_infer.py deleted file mode 100644 index 313f4caa7f61..000000000000 --- a/examples/tts/easymagpie_vllm_omni/easy_magpietts_single_infer.py +++ /dev/null @@ -1,141 +0,0 @@ -# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -Minimal pure-PyTorch single-utterance inference for EasyMagpieTTS. - -No vLLM, no manifest, no evalset config. Just: one context wav + one text -> one wav. - -Example: - python examples/tts/easy_magpietts_single_infer.py \\ - --nemo_file /path/to/EMTTS_Pretraining_Qwen_WithCrossLingual_3_5_Delay.nemo \\ - --codec_model_path /path/to/25fps_spectral_codec_with_bandwidth_extension.nemo \\ - --phoneme_tokenizer_path /path/to/bpe_ipa_tokenizer_2048_en_de_es_fr_hi_it_vi_zh.json \\ - --context_audio /path/to/reference_voice.wav \\ - --text "Hello, this is a test of the EasyMagpie text to speech model." \\ - --out_wav ./out.wav -""" -from __future__ import annotations - -import argparse - -import soundfile as sf -import torch - -from nemo.collections.tts.modules.magpietts_inference.utils import ModelLoadConfig, load_easy_magpie_model -from nemo.utils import logging - - -def main(): - parser = argparse.ArgumentParser(description="EasyMagpieTTS single-utterance pure-torch inference") - parser.add_argument("--nemo_file", required=True, help="Path to the EasyMagpieTTS .nemo checkpoint") - parser.add_argument("--codec_model_path", required=True, help="Path to the audio codec .nemo checkpoint") - parser.add_argument( - "--phoneme_tokenizer_path", - default=None, - help="Override the phoneme (IPA BPE) tokenizer path baked into the checkpoint. " - "Required if the path stored in the .nemo does not exist locally.", - ) - parser.add_argument("--context_audio", default=None, help="Reference/context wav for voice cloning") - parser.add_argument( - "--context_text", - default=None, - help="Optional style/context text tag. The voice is cloned from --context_audio; this is a " - "separate style/language conditioning string. If omitted, the correct in-distribution " - '"no text context" placeholder is auto-selected to match how the checkpoint was trained ' - "(language tag like [EN] if add_language_to_context_text=True, else [NO TEXT CONTEXT]). " - "Do NOT pass a free-form sentence unless you want it spoken/styled.", - ) - parser.add_argument( - "--language", - default="en", - help="Language of --text; used to build the [LANG] context-text placeholder for checkpoints " - "trained with add_language_to_context_text=True (e.g. en, de, es, fr, it, hi, zh, vi, ko-KR, pt-BR, ar)", - ) - parser.add_argument("--text", required=True, help="Text to synthesize") - parser.add_argument("--out_wav", default="./out.wav", help="Output wav path") - - # Tokenizer selection: defaults to the first text tokenizer in the checkpoint config - # (e.g. nemotron_nano_30b). Override only if your checkpoint has multiple. - parser.add_argument("--main_tokenizer_name", default=None) - - # The legacy Qwen EasyMagpie checkpoint was trained without CAS embeddings on context text. - parser.add_argument( - "--disable_cas_for_context_text", - action="store_true", - help="Set for legacy checkpoints trained without CAS embeddings on context text", - ) - - # Sampling / decoding parameters (defaults mirror the InferEvaluate functional test). - parser.add_argument("--temperature", type=float, default=0.6) - parser.add_argument("--topk", type=int, default=80) - parser.add_argument("--use_cfg", action="store_true", default=True) - parser.add_argument("--no_cfg", dest="use_cfg", action="store_false") - parser.add_argument("--cfg_scale", type=float, default=2.5) - parser.add_argument("--no_local_transformer", dest="use_local_transformer", action="store_false", default=True) - parser.add_argument("--max_steps", type=int, default=500) - parser.add_argument("--context_audio_duration", type=float, default=5.0) - parser.add_argument("--device", default="cuda") - - args = parser.parse_args() - - model, ckpt_name = load_easy_magpie_model( - ModelLoadConfig( - nemo_file=args.nemo_file, - codecmodel_path=args.codec_model_path, - phoneme_tokenizer_path=args.phoneme_tokenizer_path, - disable_cas_for_context_text=args.disable_cas_for_context_text, - ), - device=args.device, - ) - logging.info(f"Loaded EasyMagpieTTS checkpoint: {ckpt_name}") - logging.info(f"Available text tokenizers: {list(model.tokenizer.tokenizers.keys())}") - - # Resolve the context-text placeholder to match the training-time convention. - # The dataset uses "[]" when add_language_to_context_text=True, else "[NO TEXT CONTEXT]". - # Passing the wrong placeholder is out-of-distribution and the model may literally speak it - # (e.g. starting the audio with the word "context"). - context_text = args.context_text - if context_text is None: - if getattr(model, "add_language_to_context_text", False): - context_text = f"[{args.language.upper()}]" - else: - context_text = "[NO TEXT CONTEXT]" - logging.info(f"Using context_text={context_text!r}") - - with torch.inference_mode(): - audio, audio_lens = model.do_tts( - transcript=args.text, - context_audio_file_path=args.context_audio, - context_text=context_text, - main_tokenizer_name=args.main_tokenizer_name, - context_audio_duration=args.context_audio_duration, - use_cfg=args.use_cfg, - cfg_scale=args.cfg_scale, - use_local_transformer=args.use_local_transformer, - temperature=args.temperature, - topk=args.topk, - max_steps=args.max_steps, - ) - - audio_len = int(audio_lens[0].item()) - audio_np = audio[0, :audio_len].float().detach().cpu().numpy() - sf.write(args.out_wav, audio_np, model.output_sample_rate) - logging.info( - f"Wrote {audio_len / model.output_sample_rate:.2f}s of audio " - f"({model.output_sample_rate} Hz) to {args.out_wav}" - ) - - -if __name__ == "__main__": - main() diff --git a/examples/tts/easymagpie_vllm_omni/benchmark_model.py b/examples/tts/easymagpie_vllm_omni/scripts/benchmark_model.py similarity index 100% rename from examples/tts/easymagpie_vllm_omni/benchmark_model.py rename to examples/tts/easymagpie_vllm_omni/scripts/benchmark_model.py diff --git a/examples/tts/easymagpie_vllm_omni/benchmark_service.py b/examples/tts/easymagpie_vllm_omni/scripts/benchmark_service.py similarity index 100% rename from examples/tts/easymagpie_vllm_omni/benchmark_service.py rename to examples/tts/easymagpie_vllm_omni/scripts/benchmark_service.py diff --git a/examples/tts/easymagpie_vllm_omni/benchmark_streaming_service.py b/examples/tts/easymagpie_vllm_omni/scripts/benchmark_streaming_service.py similarity index 100% rename from examples/tts/easymagpie_vllm_omni/benchmark_streaming_service.py rename to examples/tts/easymagpie_vllm_omni/scripts/benchmark_streaming_service.py diff --git a/examples/tts/easymagpie_vllm_omni/easy_magpietts_convert_to_vllm.py b/examples/tts/easymagpie_vllm_omni/scripts/easy_magpietts_convert_to_vllm.py similarity index 99% rename from examples/tts/easymagpie_vllm_omni/easy_magpietts_convert_to_vllm.py rename to examples/tts/easymagpie_vllm_omni/scripts/easy_magpietts_convert_to_vllm.py index af5cbe720f04..6b74788aa33b 100644 --- a/examples/tts/easymagpie_vllm_omni/easy_magpietts_convert_to_vllm.py +++ b/examples/tts/easymagpie_vllm_omni/scripts/easy_magpietts_convert_to_vllm.py @@ -40,7 +40,7 @@ Example:: - python examples/tts/easymagpie_vllm_omni/easy_magpietts_convert_to_vllm.py \\ + python examples/tts/easymagpie_vllm_omni/scripts/easy_magpietts_convert_to_vllm.py \\ --nemo_file /path/to/EMTTS_SmallMamba.nemo \\ --codec_model_path /path/to/25fps_spectral_codec.nemo \\ --outdir ./easymagpie_vllm_model \\ diff --git a/examples/tts/easymagpie_vllm_omni/easymagpie_inference_demo.ipynb b/examples/tts/easymagpie_vllm_omni/scripts/easymagpie_inference_demo.ipynb similarity index 100% rename from examples/tts/easymagpie_vllm_omni/easymagpie_inference_demo.ipynb rename to examples/tts/easymagpie_vllm_omni/scripts/easymagpie_inference_demo.ipynb diff --git a/examples/tts/easymagpie_vllm_omni/export_codec_decoder_onnx.py b/examples/tts/easymagpie_vllm_omni/scripts/export_codec_decoder_onnx.py similarity index 99% rename from examples/tts/easymagpie_vllm_omni/export_codec_decoder_onnx.py rename to examples/tts/easymagpie_vllm_omni/scripts/export_codec_decoder_onnx.py index 85098993daad..6c7382bd2428 100644 --- a/examples/tts/easymagpie_vllm_omni/export_codec_decoder_onnx.py +++ b/examples/tts/easymagpie_vllm_omni/scripts/export_codec_decoder_onnx.py @@ -49,7 +49,7 @@ Stage 2 (TRT engine build) lives in ``export_codec_decoder_trt.py``. Example: - python examples/tts/easymagpie_vllm_omni/export_codec_decoder_onnx.py \\ + python examples/tts/easymagpie_vllm_omni/scripts/export_codec_decoder_onnx.py \\ --codec_model_path /path/to/25fps_spectral_codec_with_bandwidth_extension.nemo \\ --nemo_file /path/to/easymagpie.nemo \\ --onnx-path codec/codec_decoder.onnx \\ diff --git a/examples/tts/easymagpie_vllm_omni/export_codec_decoder_trt.py b/examples/tts/easymagpie_vllm_omni/scripts/export_codec_decoder_trt.py similarity index 97% rename from examples/tts/easymagpie_vllm_omni/export_codec_decoder_trt.py rename to examples/tts/easymagpie_vllm_omni/scripts/export_codec_decoder_trt.py index 67d2b0174107..27a1ff8cfd0e 100644 --- a/examples/tts/easymagpie_vllm_omni/export_codec_decoder_trt.py +++ b/examples/tts/easymagpie_vllm_omni/scripts/export_codec_decoder_trt.py @@ -23,7 +23,7 @@ profile flags. Example: - python examples/tts/easymagpie_vllm_omni/export_codec_decoder_trt.py \\ + python examples/tts/easymagpie_vllm_omni/scripts/export_codec_decoder_trt.py \\ --onnx-path codec/codec_decoder.onnx \\ --trt-path codec/codec_decoder.plan \\ --batch-profile 1 8 32 \\ diff --git a/examples/tts/easymagpie_vllm_omni/run_service_request.ipynb b/examples/tts/easymagpie_vllm_omni/scripts/run_service_request.ipynb similarity index 100% rename from examples/tts/easymagpie_vllm_omni/run_service_request.ipynb rename to examples/tts/easymagpie_vllm_omni/scripts/run_service_request.ipynb diff --git a/examples/tts/easymagpie_vllm_omni/scripts/streaming_codec_decode_demo.ipynb b/examples/tts/easymagpie_vllm_omni/scripts/streaming_codec_decode_demo.ipynb new file mode 100644 index 000000000000..a4b05ca56c33 --- /dev/null +++ b/examples/tts/easymagpie_vllm_omni/scripts/streaming_codec_decode_demo.ipynb @@ -0,0 +1,599 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "ae5000ca", + "metadata": {}, + "source": [ + "# Streaming codec decode demo (`streaming_codec_decode.py`)\n", + "\n", + "This notebook tests **chunked / overlap-save** decoding of EasyMagpieTTS codec\n", + "codes against a **full one-shot** decode, using the causal `25fps spectral codec`.\n", + "\n", + "**Streaming schedule requested:** `left_context = 24` past frames + `chunk_frames = 3`\n", + "new (\"effective\") frames per step = a **27-frame** decode window that shifts\n", + "forward 3 frames at a time. Because the decoder is causal, `right_context = 0`\n", + "(no lookahead).\n", + "\n", + "We then:\n", + "1. Decode the **entire** code sequence in one shot.\n", + "2. Decode it **incrementally** (3 frames / step), concatenating emitted audio.\n", + "3. **Plot** both waveforms + their difference.\n", + "4. **Listen** to both.\n", + "\n", + "Provide your codes as a torch file (`torch.save(codes, \"codes.pt\")`) with shape\n", + "`(C, T)` or `(B, C, T)` in the model's codebook space (i.e. the same space as\n", + "`StreamingFinalizeOutput.audio_codes`)." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "3a425dfb", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "device: cuda\n" + ] + } + ], + "source": [ + "import os\n", + "import sys\n", + "\n", + "import numpy as np\n", + "import torch\n", + "\n", + "# ----------------------------------------------------------------------------\n", + "# CONFIG -- edit these paths.\n", + "# ----------------------------------------------------------------------------\n", + "# Directory containing streaming_codec_decode.py (this notebook's folder).\n", + "SCRIPT_DIR = os.path.abspath(\".\")\n", + "\n", + "# Checkpoints.\n", + "NEMO_FILE = \"/home/vklimkov/workspace/emp/ckpt/easymagpietts_NEXT/2605_NemotronTTS_V0.2/v2/2605_EMTTS_SmallMamba_Step150k_posttrained_epoch12.nemo\" # <-- set me\n", + "CODEC_MODEL_PATH = \"/home/vklimkov/workspace/emp/ckpt/easymagpietts_NEXT/25fps_spectral_codec_with_bandwidth_extension.nemo\"\n", + "PHONEME_TOKENIZER_PATH = \"/home/vklimkov/workspace/emp/ckpt/easymagpietts_NEXT/2605_NemotronTTS_V0.2/bpe_ipa_tokenizer_2048_en_de_es_fr_hi_it_vi_zh_ko-KR_pt-BR_ar.json\" # optional override; set if the path baked in the .nemo is missing\n", + "DISABLE_CAS_FOR_CONTEXT_TEXT = False # set True for legacy Qwen EasyMagpie checkpoints\n", + "\n", + "# Codes to decode (torch.save'd tensor or dict). Shape (C, T) or (B, C, T).\n", + "CODES_PATH = \"/home/vklimkov/workspace/emp/NeMo/examples/tts/easymagpie_vllm_omni/codes.pt\" # <-- set me\n", + "\n", + "# Streaming schedule: 24 left-context + 3 effective frames/step (= 27-frame window), R=0.\n", + "LEFT_CONTEXT = 24\n", + "CHUNK_FRAMES = 3\n", + "RIGHT_CONTEXT = 0\n", + "\n", + "DEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n", + "\n", + "sys.path.insert(0, SCRIPT_DIR)\n", + "from streaming_codec_decode import ( # noqa: E402\n", + " StreamingCodecDecoder,\n", + " StreamingDecodeConfig,\n", + " chunked_decode,\n", + " make_nemo_decode_fn,\n", + ")\n", + "\n", + "print(\"device:\", DEVICE)" + ] + }, + { + "cell_type": "markdown", + "id": "d2ef28f3", + "metadata": {}, + "source": [ + "## 1. Load the model + codec\n", + "\n", + "`load_easy_magpie_model` restores the decoder-only model **and** the codec, sets\n", + "up the optional index-converter and frame-unstacking, and gives us\n", + "`model._codec_helper` used by `make_nemo_decode_fn`." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "a35ee501", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/home/vklimkov/miniconda3/envs/emp/lib/python3.12/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n", + "OneLogger: Setting error_handling_strategy to DISABLE_QUIETLY_AND_REPORT_METRIC_ERROR for rank (rank=0) with OneLogger disabled. To override: explicitly set error_handling_strategy parameter.\n", + "No exporters were provided. This means that no telemetry data will be collected.\n", + "[NeMo W 2026-06-04 17:19:53 evaluate_generated_audio:48] UTMOSv2Calculator not available: UTMOSv2 is not installed. Please install it using `pip install git+https://github.com/sarulab-speech/UTMOSv2.git@v1.2.1`.. UTMOSv2 metrics will be disabled. Install required dependencies to enable.To install utmosv2 run `pip install git+https://github.com/sarulab-speech/UTMOSv2.git@v1.2.1`.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[NeMo I 2026-06-04 17:19:53 utils:452] Loading model from NeMo archive: /home/vklimkov/workspace/emp/ckpt/easymagpietts_NEXT/2605_NemotronTTS_V0.2/v2/2605_EMTTS_SmallMamba_Step150k_posttrained_epoch12.nemo\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[NeMo W 2026-06-04 17:19:55 modelPT:176] If you intend to do training or fine-tuning, please call the ModelPT.setup_training_data() method and provide a valid configuration file to setup the train data loader.\n", + " Train config : \n", + " dataset:\n", + " dataset_type: tarred_vocoder\n", + " dataset_args:\n", + " dataset_meta:\n", + " libritts:\n", + " manifest_path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/libritts_audio/22khz/tarred_audio/train_manifest.json\n", + " tar_filepath: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/libritts_audio/22khz/tarred_audio/audio_{0..44}.tar\n", + " librittsr:\n", + " manifest_path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/libritts_r_audio/22khz/tarred_audio/train_manifest.json\n", + " tar_filepath: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/libritts_r_audio/22khz/tarred_audio/audio_{0..49}.tar\n", + " cv:\n", + " manifest_path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/commonvoice13/22khz/tarred_audio/train_manifest.json\n", + " tar_filepath: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/commonvoice13/22khz/tarred_audio/audio_{0..279}.tar\n", + " hifitts2:\n", + " manifest_path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/hifitts2/22khz/tarred_audio/train_manifest.json\n", + " tar_filepath: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/hifitts2/22khz/tarred_audio/audio_{0..3199}.tar\n", + " nvyt40k:\n", + " manifest_path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/nvyt40k_audio/22khz/tarred_audio/train_manifest.json\n", + " tar_filepath: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/nvyt40k_audio/22khz/tarred_audio/audio_{0..3799}.tar\n", + " sample_rate: 22050\n", + " n_samples: 17640\n", + " min_duration: 0.5\n", + " max_duration: null\n", + " shard_strategy: replicate\n", + " sample_type: weighted_random\n", + " sample_args:\n", + " batch_size: 80\n", + " steps_per_epoch: 8000\n", + " dataset_weights:\n", + " - 0.02\n", + " - 0.02\n", + " - 0.1\n", + " - 0.26\n", + " - 0.6\n", + " shuffle_n: 20000\n", + " dataloader_params:\n", + " batch_size: 80\n", + " drop_last: true\n", + " num_workers: 4\n", + " \n", + "[NeMo W 2026-06-04 17:19:55 modelPT:183] If you intend to do validation, please call the ModelPT.setup_validation_data() or ModelPT.setup_multiple_validation_data() method and provide a valid configuration file to setup the validation data loader(s). \n", + " Validation config : \n", + " dataset:\n", + " dataset_type: vocoder\n", + " dataset_args:\n", + " sample_rate: 22050\n", + " n_samples: null\n", + " min_duration: null\n", + " max_duration: null\n", + " trunc_duration: 10.0\n", + " dataset_meta:\n", + " libritts:\n", + " manifest_path: /tmp/rlangman/data/libritts/val_manifest.json\n", + " audio_dir: /tmp/rlangman/data/libritts/audio_22khz\n", + " librittsr:\n", + " manifest_path: /tmp/rlangman/data/librittsr/val_manifest.json\n", + " audio_dir: /tmp/rlangman/data/librittsr/audio_22khz\n", + " cv:\n", + " manifest_path: /tmp/rlangman/data/cv/val_manifest.json\n", + " audio_dir: /tmp/rlangman/data/cv/audio_22khz\n", + " hifitts2:\n", + " manifest_path: /tmp/rlangman/data/hifitts2/val_manifest.json\n", + " audio_dir: /tmp/rlangman/data/hifitts2/audio_22khz\n", + " nvyt40k:\n", + " manifest_path: /tmp/rlangman/data/nvyt40k/val_manifest.json\n", + " audio_dir: /tmp/rlangman/data/nvyt40k/audio_22khz\n", + " dataloader_params:\n", + " batch_size: 4\n", + " num_workers: 2\n", + " \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[NeMo I 2026-06-04 17:19:55 audio_codec:109] Vector quantizer does not support commit loss.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Loading weights: 100%|██████████| 248/248 [00:00<00:00, 30449.56it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[NeMo I 2026-06-04 17:19:57 save_restore_connector:286] Model AudioCodecModel was successfully restored from /home/vklimkov/workspace/emp/ckpt/easymagpietts_NEXT/25fps_spectral_codec_with_bandwidth_extension.nemo.\n", + "[NeMo I 2026-06-04 17:19:57 easy_magpietts_inference:317] Multi-mode training with 1 modes:\n", + "[NeMo I 2026-06-04 17:19:57 easy_magpietts_inference:319] - streaming_3_5: text_input_mode=streaming, streaming_phonemes_delay=3, streaming_speech_delay=5\n", + "[NeMo I 2026-06-04 17:19:59 easy_magpietts_inference:436] Using decoder type: nemotron_h\n", + "[NeMo I 2026-06-04 17:20:06 easy_magpietts_inference:477] NemotronH config: 31 layers, pattern=MEMEM*EMEMEM*EMEMEME...\n", + "[NeMo I 2026-06-04 17:20:06 easy_magpietts_inference:504] Single training mode 'streaming_3_5', skipping task embedding\n", + "[NeMo I 2026-06-04 17:20:06 easy_magpietts_inference:542] Local transformer type: autoregressive\n", + "[NeMo I 2026-06-04 17:20:08 save_restore_connector:286] Model EasyMagpieTTSInferenceModel was successfully restored from /home/vklimkov/workspace/emp/ckpt/easymagpietts_NEXT/2605_NemotronTTS_V0.2/v2/2605_EMTTS_SmallMamba_Step150k_posttrained_epoch12.nemo.\n", + "[NeMo I 2026-06-04 17:20:08 utils:473] EasyMagpieTTS model loaded and ready for inference.\n", + "checkpoint: 2605_EMTTS_SmallMamba_Step150k_posttrained_epoch12\n", + "output_sample_rate: 22050\n", + "decoder: ResNetDecoder | is_causal: None\n", + "num_audio_codebooks (model space): 8\n", + "frame_stacking_factor: 2\n", + "codec native num_codebooks: 5\n" + ] + } + ], + "source": [ + "from nemo.collections.tts.modules.magpietts_inference.utils import ModelLoadConfig, load_easy_magpie_model\n", + "\n", + "model, ckpt_name = load_easy_magpie_model(\n", + " ModelLoadConfig(\n", + " nemo_file=NEMO_FILE,\n", + " codecmodel_path=CODEC_MODEL_PATH,\n", + " phoneme_tokenizer_path=PHONEME_TOKENIZER_PATH,\n", + " disable_cas_for_context_text=DISABLE_CAS_FOR_CONTEXT_TEXT,\n", + " ),\n", + " device=DEVICE,\n", + ")\n", + "model.eval()\n", + "\n", + "SR = int(model.output_sample_rate)\n", + "decoder = model._codec_model.audio_decoder\n", + "print(\"checkpoint:\", ckpt_name)\n", + "print(\"output_sample_rate:\", SR)\n", + "print(\"decoder:\", type(decoder).__name__, \"| is_causal:\", getattr(decoder, \"is_causal\", None))\n", + "print(\"num_audio_codebooks (model space):\", model.num_audio_codebooks)\n", + "print(\"frame_stacking_factor:\", model.frame_stacking_factor)\n", + "print(\"codec native num_codebooks:\", model._codec_model.num_codebooks)" + ] + }, + { + "cell_type": "markdown", + "id": "273588c1", + "metadata": {}, + "source": [ + "## 2. Load the codes\n", + "\n", + "Robustly coerce whatever you saved into a `(1, C, T)` long tensor. If your codes\n", + "are arranged `(T, C)` instead of `(C, T)`, flip `ASSUME_TIME_LAST` below." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "9f6941f8", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "codes: (1, 8, 148) torch.int64 | frames: 148\n", + "duration @25fps: 5.92s | code value range: [0, 1021]\n" + ] + } + ], + "source": [ + "ASSUME_TIME_LAST = True # codes are (..., C, T); set False if they are (..., T, C)\n", + "\n", + "raw = torch.load(CODES_PATH, map_location=\"cpu\")\n", + "if isinstance(raw, dict):\n", + " for k in (\"codes\", \"audio_codes\", \"predicted_codes\", \"tokens\"):\n", + " if k in raw:\n", + " print(f\"using dict key {k!r}\")\n", + " raw = raw[k]\n", + " break\n", + " else:\n", + " raise KeyError(f\"none of codes/audio_codes/predicted_codes/tokens in {list(raw.keys())}\")\n", + "\n", + "codes = raw if torch.is_tensor(raw) else torch.as_tensor(raw)\n", + "if codes.dim() == 2:\n", + " codes = codes.unsqueeze(0) # (C, T) -> (1, C, T)\n", + "assert codes.dim() == 3, f\"expected (B, C, T), got {tuple(codes.shape)}\"\n", + "if not ASSUME_TIME_LAST:\n", + " codes = codes.transpose(1, 2).contiguous() # (B, T, C) -> (B, C, T)\n", + "\n", + "# Keep a single utterance for the demo.\n", + "codes = codes[:1].long().to(DEVICE)\n", + "n_frames = codes.shape[-1]\n", + "codes_len = torch.tensor([n_frames], dtype=torch.long, device=DEVICE)\n", + "\n", + "print(\"codes:\", tuple(codes.shape), codes.dtype, \"| frames:\", n_frames)\n", + "print(f\"duration @25fps: {n_frames / 25:.2f}s | code value range: [{int(codes.min())}, {int(codes.max())}]\")" + ] + }, + { + "cell_type": "markdown", + "id": "7b6f1b1b", + "metadata": {}, + "source": [ + "## 3. Full one-shot decode (reference)\n", + "\n", + "`make_nemo_decode_fn` reproduces exactly what `streaming_finalize` does:\n", + "optional frame-unstack -> optional index-converter -> `codec.decode`." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "93494767", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "full decode: (130536,) samples | 5.92s\n", + "inferred output samples/frame: 882\n" + ] + } + ], + "source": [ + "decode_fn = make_nemo_decode_fn(model)\n", + "\n", + "with torch.inference_mode():\n", + " full_audio, full_len = decode_fn(codes, codes_len)\n", + "\n", + "full_wav = full_audio[0, : int(full_len[0])].float().cpu().numpy()\n", + "print(\"full decode:\", full_wav.shape, \"samples |\", f\"{full_wav.shape[0] / SR:.2f}s\")\n", + "print(\"inferred output samples/frame:\", full_wav.shape[0] // n_frames)" + ] + }, + { + "cell_type": "markdown", + "id": "cd7f35ee", + "metadata": {}, + "source": [ + "## 4. Streaming decode: 24 context + 3 frames/step, shifting forward\n", + "\n", + "We feed the codes **3 frames at a time** (simulating the AR model emitting frames\n", + "as it goes). The `StreamingCodecDecoder` buffers them and, for each new 3-frame\n", + "body, decodes a `[<=24 left context | 3 body]` window and keeps only the 3-frame\n", + "body audio. `flush()` drains the final partial chunk." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "20a26cc3", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "streaming config: StreamingDecodeConfig(chunk_frames=3, left_context=24, right_context=0, samples_per_frame=None) | decode window = 27 frames\n", + "pushed 50 steps of 3 frames; tail flush = 0 samples\n", + "streaming decode: (6174,) samples | 0.28s\n" + ] + } + ], + "source": [ + "# IMPORTANT: set samples_per_frame explicitly. The codec helper zero-pads windows\n", + "# shorter than 4 frames, so inferring spf from the first (3-frame) window would be\n", + "# wrong and truncate the stream. Take the exact value from the full decode.\n", + "SPF = int(full_len[0]) // n_frames\n", + "cfg = StreamingDecodeConfig(\n", + " chunk_frames=CHUNK_FRAMES,\n", + " left_context=LEFT_CONTEXT,\n", + " right_context=RIGHT_CONTEXT,\n", + " samples_per_frame=SPF,\n", + ")\n", + "print(\"samples_per_frame:\", SPF)\n", + "print(\"streaming config:\", cfg, \"| decode window =\", cfg.left_context + cfg.chunk_frames, \"frames\")\n", + "\n", + "dec = StreamingCodecDecoder(decode_fn, cfg)\n", + "\n", + "emitted_pieces = []\n", + "step_log = []\n", + "with torch.inference_mode():\n", + " for start in range(0, n_frames, CHUNK_FRAMES):\n", + " new_frames = codes[:, :, start : start + CHUNK_FRAMES] # (1, C, <=3)\n", + " out = dec.push(new_frames) # (1, T_emit)\n", + " if out.shape[-1] > 0:\n", + " emitted_pieces.append(out)\n", + " step_log.append((start, new_frames.shape[-1], out.shape[-1]))\n", + " tail = dec.flush()\n", + " if tail.shape[-1] > 0:\n", + " emitted_pieces.append(tail)\n", + "\n", + "stream_audio = torch.cat(emitted_pieces, dim=-1)\n", + "stream_wav = stream_audio[0].float().cpu().numpy()\n", + "print(f\"pushed {len(step_log)} steps of {CHUNK_FRAMES} frames; tail flush = {tail.shape[-1]} samples\")\n", + "print(\"streaming decode:\", stream_wav.shape, \"samples |\", f\"{stream_wav.shape[0] / SR:.2f}s\")" + ] + }, + { + "cell_type": "markdown", + "id": "237f1098", + "metadata": {}, + "source": [ + "## 5. Compare: waveforms + difference\n", + "\n", + "For a causal decoder with `left_context` >= receptive field, streaming should\n", + "match the one-shot decode to within float tolerance. We align lengths, report the\n", + "max/RMS difference, and plot both overlaid plus the error." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "32be751a", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "length: full=130536 stream=6174 (compared first 6174)\n", + "max |diff| = 2.185e-03 RMS diff = 2.128e-04\n", + "signal RMS = 9.284e-04\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAABWUAAAMWCAYAAACZdeR/AAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjksIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvJkbTWQAAAAlwSFlzAAAPYQAAD2EBqD+naQABAABJREFUeJzs3XecFPXdB/DPzGy9fsfd0aUJIhZQVKKxS4LGRmKsSTTGkv5oMJpYsSUk9h4ssSY+lmhIHkWQagEEaQpI79xx/W53b9vU548p269xR/28Xy9eerOzO2XnNzvzne/v+xMMwzBARERERERERERERHuFuK9XgIiIiIiIiIiIiOhQwqAsERERERERERER0V7EoCwRERERERERERHRXsSgLBEREREREREREdFexKAsERERERERERER0V7EoCwRERERERERERHRXsSgLBEREREREREREdFexKAsERERERERERER0V7EoCwRERERERERERHRXsSgLBERERF12Pz58yEIAubPn+9M++lPf4rBgwfvs3XqSa+++ioEQcDSpUv39apkEAQB9957775ejZz29/UjIiIi2pcYlCUiIiI6yNmBxWz//vjHP+7r1TvkvPnmm3jiiSf29WoQERER0T7k2tcrQERERER7x/33348hQ4akTDv66KP30docut58802sXr0aN998875eFSIiIiLaRxiUJSIiIjpEnHfeeTjhhBP29WoQERERER3yWL6AiIiIiHLW/xw8eDB++tOfdttynnvuORx11FHwer3o168ffv3rX6OlpSVlnjPPPBNHH300vvnmG5x11lnIy8tD//798dBDD2V8Xjwex+TJk3H44YfD6/Vi4MCBuO222xCPxzu0Pm+99RbGjh2LwsJCFBUV4ZhjjsGTTz6ZdTmTJk1CRUUF8vPz8f3vfx/19fWd3r4zzzwTH374IbZv3+6UkGivHm88Hsfvfvc7VFRUoLCwEBdddBF27dqVdd6qqir87Gc/Q+/eveH1enHUUUfh5ZdfzpgvFovh3nvvxYgRI+Dz+dC3b1/84Ac/wObNm515wuEwbrnlFgwcOBBerxdHHHEEHnnkERiG0ePrR0RERHSwY6YsERER0SEiEAigoaEhZVp5efleW/69996L++67D+PHj8cvf/lLrF+/Hn/729/w5ZdfYsGCBXC73c68zc3NOPfcc/GDH/wAl112Gf71r3/hD3/4A4455hicd955AABd13HRRRfh888/x4033ogjjzwSq1atwuOPP44NGzZg2rRpba7PrFmzcOWVV+Kcc87BX//6VwDA2rVrsWDBAtx0000p8/72t79FaWkpJk+ejG3btuGJJ57Ab37zG7z99tud2r4777wTgUAAu3btwuOPPw4AKCgoaHM9r7/+evzjH//AVVddhVNOOQVz587F+eefnzFfbW0tvvWtb0EQBPzmN79BRUUFPvroI1x33XUIBoNOuQRN03DBBRdgzpw5uOKKK3DTTTchFAph1qxZWL16NYYNGwbDMHDRRRdh3rx5uO666zBmzBjMnDkTt956K6qqqpx174n1IyIiIjokGERERER0UHvllVcMAFn/2QAYkydPznjvoEGDjGuuucb5e968eQYAY968ec60a665xhg0aFCb61BXV2d4PB7ju9/9rqFpmjP9mWeeMQAYL7/8sjPtjDPOMAAYr7/+ujMtHo8bffr0MS655BJn2htvvGGIomh89tlnKcuaOnWqAcBYsGBBm+t00003GUVFRYaqqjnnsffd+PHjDV3Xnem/+93vDEmSjJaWlk5v3/nnn9/u/rKtXLnSAGD86le/Spl+1VVXZXxn1113ndG3b1+joaEhZd4rrrjCKC4uNiKRiGEYhvHyyy8bAIzHHnssY3n2Nk6bNs0AYDz44IMpr//whz80BEEwNm3a1GPrR0RERHQoYPkCIiIiokPEs88+i1mzZqX821tmz54NWZZx8803QxQTl6A33HADioqK8OGHH6bMX1BQgB//+MfO3x6PByeddBK2bNniTHv33Xdx5JFHYuTIkWhoaHD+nX322QCAefPmtblOJSUlCIfDHdoPN954IwRBcP4+7bTToGkatm/f3qXt66jp06cDAP7nf/4nZXp6VqlhGHjvvfdw4YUXwjCMlP0xYcIEBAIBLF++HADw3nvvoby8HL/97W8zlmdv4/Tp0yFJUsZyb7nlFhiGgY8++qjH1o+IiIjoUMDyBURERESHiJNOOmmfDfRlBy+POOKIlOkejwdDhw51XrcNGDAgJQgKAKWlpfj666+dvzdu3Ii1a9eioqIi6zLr6uoAAE1NTZBl2Znu9/tRXFyMX/3qV3jnnXdw3nnnoX///vjud7+Lyy67DOeee27GZx122GEZ6wKYZRa6sn0dtX37doiiiGHDhqVMT19OfX09Wlpa8MILL+CFF17I+ln2/ti8eTOOOOIIuFy5bwW2b9+Ofv36obCwMGX6kUce6bzeU+tHREREdChgUJaIiIiIctI0bZ8sV5KkrNONpEGmdF3HMcccg8ceeyzrvAMHDgQA/OAHP8Ann3ziTL/mmmvw6quvorKyEitXrsTMmTPx0Ucf4aOPPsIrr7yCq6++Gq+99lqn12df0nUdAPDjH/8Y11xzTdZ5jj322L25Sin29/UjIiIi2tsYlCUiIiIilJaWoqWlJWWaLMvYvXt3t3z+oEGDAADr16/H0KFDU5axdetWjB8/vtOfOWzYMHz11Vc455xzMrJqkz366KNORisA9OvXz/l/j8eDCy+8EBdeeCF0XcevfvUrPP/887j77rtx+OGHd3hdOrN9ba1rts/Vdd3JbrWtX78+Zb6KigoUFhZC07R29+WwYcOwePFiKIqSMrha+nJnz56NUCiUki27bt065/WeWj8iIiKiQwFryhIRERERhg0bhk8//TRl2gsvvNBtmbLjx4+Hx+PBU089lZJd+ve//x2BQADnn39+pz/zsssuQ1VVFV588cWM16LRKMLhMABg7NixGD9+vPNv1KhRAIDGxsaU94ii6GRrxuPxTq1LZ7YvPz8fgUCgQ5973nnnAQCeeuqplOlPPPFEyt+SJOGSSy7Be++9h9WrV2d8Tn19vfP/l1xyCRoaGvDMM89kzGev+/e+9z1ompYxz+OPPw5BEJz16on1IyIiIjoUMFOWiIiIiHD99dfjF7/4BS655BJ85zvfwVdffYWZM2eivLy8Wz6/oqICt99+O+677z6ce+65uOiii7B+/Xo899xzOPHEE1MG9eqon/zkJ3jnnXfwi1/8AvPmzcO3v/1taJqGdevW4Z133sHMmTPbrKF7/fXXo6mpCWeffTYGDBiA7du34+mnn8aYMWOc2qk9sX1jx47F22+/jUmTJuHEE09EQUEBLrzwwqyfO2bMGFx55ZV47rnnEAgEcMopp2DOnDnYtGlTxrx/+ctfMG/ePIwbNw433HADRo0ahaamJixfvhyzZ89GU1MTAODqq6/G66+/jkmTJmHJkiU47bTTEA6HMXv2bPzqV7/CxRdfjAsvvBBnnXUW7rzzTmzbtg2jR4/Gxx9/jP/85z+4+eabnRqyPbF+RERERIcCBmWJiIiICDfccAO2bt2Kv//975gxYwZOO+00zJo1C+ecc063LePee+9FRUUFnnnmGfzud79DWVkZbrzxRvz5z3/O2Y2+LaIoYtq0aXj88cfx+uuv49///jfy8vIwdOhQ3HTTTRgxYkSb7//xj3+MF154Ac899xxaWlrQp08fXH755bj33nship3vUNbR7fvVr36FlStX4pVXXsHjjz+OQYMG5QzKAsDLL7+MiooK/POf/8S0adNw9tln48MPP3Rq5tp69+6NJUuW4P7778f777+P5557Dr169cJRRx2Fv/71r858kiRh+vTp+NOf/oQ333wT7733Hnr16oVTTz0VxxxzjLNv//vf/+Kee+7B22+/jVdeeQWDBw/Gww8/jFtuuaVH14+IiIjoUCAY+8voBERERERERERERESHANaUJSIiIiIiIiIiItqLGJQlIiIiIiIiIiIi2osYlCUiIiIiIiIiIiLaixiUJSIiIiIiIiIiItqLGJQlIiIiIiIiIiIi2osYlCUiIiIiIiIiIiLai1z7egX2V7quo7q6GoWFhRAEYV+vDhEREREREREREe3HDMNAKBRCv379IIpt58IyKJtDdXU1Bg4cuK9Xg4iIiIiIiIiIiA4gO3fuxIABA9qch0HZHAoLCwGYO7GoqGgfr03P0nUd9fX1qKioaDeKT0S5sS0RdQ+2JaLuwbZE1H3Ynoi6B9sSHeyCwSAGDhzoxBXbwqBsDnbJgqKiokMiKBuLxVBUVMSTItEeYFsi6h5sS0Tdg22JqPuwPRF1D7YlOlR0pBQqWwARERERERERERHRXrRXgrLPPvssBg8eDJ/Ph3HjxmHJkiVtzv/uu+9i5MiR8Pl8OOaYYzB9+vSU1w3DwD333IO+ffvC7/dj/Pjx2LhxY8o8GzZswMUXX4zy8nIUFRXh1FNPxbx587p924iIiIiIiIiIiIg6o8eDsm+//TYmTZqEyZMnY/ny5Rg9ejQmTJiAurq6rPMvXLgQV155Ja677jqsWLECEydOxMSJE7F69WpnnoceeghPPfUUpk6disWLFyM/Px8TJkxALBZz5rngggugqirmzp2LZcuWYfTo0bjgggtQU1PT05tMRERERERERERElJNgGIbRkwsYN24cTjzxRDzzzDMAzPohAwcOxG9/+1v88Y9/zJj/8ssvRzgcxgcffOBM+9a3voUxY8Zg6tSpMAwD/fr1wy233ILf//73AIBAIIDevXvj1VdfxRVXXIGGhgZUVFTg008/xWmnnQYACIVCKCoqwqxZszB+/Ph21zsYDKK4uBiBQOCQqClbV1eHyspK1nQh2gNsS0Tdg22JqHuwLRF1H7Ynou7BtkQHu87EE3t0oC9ZlrFs2TLcfvvtzjRRFDF+/HgsWrQo63sWLVqESZMmpUybMGECpk2bBgDYunUrampqUgKrxcXFGDduHBYtWoQrrrgCvXr1whFHHIHXX38dxx9/PLxeL55//nlUVlZi7NixWZcbj8cRj8edv4PBIADzhKHrepe2/0Ch6zoMwzjot5Oop7EtEXUPtiWi7sG2RNR92J6IugfbEh3sOnNs92hQtqGhAZqmoXfv3inTe/fujXXr1mV9T01NTdb57bID9n/bmkcQBMyePRsTJ05EYWEhRFFEZWUlZsyYgdLS0qzLnTJlCu67776M6fX19SllEQ5Guq4jEAjAMAw+qSLaA2xLRN2DbYmoe7AtEXUftiei7sG2RAe7UCjU4Xl7NCi7rxiGgV//+teorKzEZ599Br/fj5deegkXXnghvvzyS/Tt2zfjPbfffntKhm4wGMTAgQNRUVFxSJQvEAQBFRUVHTop7myKYOn2Znz/uP57Ye2IDhydbUtElB3bElH3YFsi6j5sT0Tdg22JDnY+n6/D8/ZoULa8vBySJKG2tjZlem1tLfr06ZP1PX369Glzfvu/tbW1KcHV2tpajBkzBgAwd+5cfPDBB2hubnYCqs899xxmzZqF1157LWstW6/XC6/XmzFdFMVD4kQhCEKHt3XW2jo88vF6XDJ24F5YM6IDS2faEhHlxrZE1D3Yloi6D9sTUfdgW6KDWWeO6x5tAR6PB2PHjsWcOXOcabquY86cOTj55JOzvufkk09OmR8AZs2a5cw/ZMgQ9OnTJ2WeYDCIxYsXO/NEIhEAmTtCFEXWLekmkiDs61UgIiIiIiIiIiI6IPV4+YJJkybhmmuuwQknnICTTjoJTzzxBMLhMK699loAwNVXX43+/ftjypQpAICbbroJZ5xxBh599FGcf/75eOutt7B06VK88MILAMwnKjfffDMefPBBDB8+HEOGDMHdd9+Nfv36YeLEiQDMwG5paSmuueYa3HPPPfD7/XjxxRexdetWnH/++T29yQc9w9jXa0BERERERERERHTg6vGg7OWXX476+nrcc889qKmpwZgxYzBjxgxnoK4dO3akZLSecsopePPNN3HXXXfhjjvuwPDhwzFt2jQcffTRzjy33XYbwuEwbrzxRrS0tODUU0/FjBkznLoN5eXlmDFjBu68806cffbZUBQFRx11FP7zn/9g9OjRPb3JBz1F1+F2sZsBERERERERERFRVwiGwbzHbILBIIqLixEIBA6Jgb7q6upQWVnZodoXj368Hu8t24WFt5+zF9aO6MDR2bZERNmxLRF1D7Ylou7D9kTUPdiW6GDXmXgiWwB1WlzV4XVL+3o1iIiIiIiIiIiIDkgMylKnyaqOrQ1hvLl4x75eFSIiIiIiIiIiogMOg7LUaXFVBwB8uKp6H68JERERERERERHRgYdBWeo02QrKshoxERERERERERFR57n29QrQgUfWdJx6eDlK8z37elWIiIiIiIiIiIgOOMyUpU6TVQ2PXDoaeRzsi4iIiIiIiIiIqNMYlKVOUzUD+V4JcVXb16tCRERERERERER0wGFQlrrE55YQU/R9vRpEREREREREREQHHAZlqUvckghVZ1CWiIiIiIiIiIiosxiUpT0g7OsVICIiIiIiIiIiOuAwKEtERERERERERES0FzEoS3vA2NcrQEREREREREREdMBhUJaIiIiIiIiIiIhoL2JQlvYAa8oSERERERERERF1FoOy1GkCY7FERERERERERERdxqAsERERERERERER0V7EoCwRERERERERERHRXsSgLBEREREREREREdFexKAsdYqmGxBZVJaIiIiIiIiIiKjLGJSlTlE0HW4XDxsiIiIiIiIiIqKuYnSNOkXWdHgkHjZERERERERERERdtVeia88++ywGDx4Mn8+HcePGYcmSJW3O/+6772LkyJHw+Xw45phjMH369JTXDcPAPffcg759+8Lv92P8+PHYuHFjxud8+OGHGDduHPx+P0pLSzFx4sTu3KxDkqLqcEssX0BERERERERERNRVPR6UffvttzFp0iRMnjwZy5cvx+jRozFhwgTU1dVlnX/hwoW48sorcd1112HFihWYOHEiJk6ciNWrVzvzPPTQQ3jqqacwdepULF68GPn5+ZgwYQJisZgzz3vvvYef/OQnuPbaa/HVV19hwYIFuOqqq3p6cw9av3/3K4RiChTNgIuZskRERERERERERF3W49G1xx57DDfccAOuvfZajBo1ClOnTkVeXh5efvnlrPM/+eSTOPfcc3HrrbfiyCOPxAMPPIDjjz8ezzzzDAAzS/aJJ57AXXfdhYsvvhjHHnssXn/9dVRXV2PatGkAAFVVcdNNN+Hhhx/GL37xC4wYMQKjRo3CZZdd1tObe9D617JdaA4ruPPfq1i+gIiIiIiIiIiIaA+4evLDZVnGsmXLcPvttzvTRFHE+PHjsWjRoqzvWbRoESZNmpQybcKECU7AdevWraipqcH48eOd14uLizFu3DgsWrQIV1xxBZYvX46qqiqIoojjjjsONTU1GDNmDB5++GEcffTRWZcbj8cRj8edv4PBIABA13Xout6l7T9Q6LoOwzDa3U5F0zBnXR2uP3UIdF2HKACKqkESWc6ACOh4WyKitrEtEXUPtiWi7sP2RNQ92JboYNeZY7tHg7INDQ3QNA29e/dOmd67d2+sW7cu63tqamqyzl9TU+O8bk/LNc+WLVsAAPfeey8ee+wxDB48GI8++ijOPPNMbNiwAWVlZRnLnTJlCu67776M6fX19SllEQ5Guq4jEAjAMAyIYu4s2LqGBgBAqDWMuro6aIqM6ppaeF3MnCUCOt6WiKhtbEtE3YNtiaj7sD0RdQ+2JTrYhUKhDs/bo0HZfcWOSt9555245JJLAACvvPIKBgwYgHfffRc///nPM95z++23p2ToBoNBDBw4EBUVFSgqKto7K76P6LoOQRBQUVHR5kmxuMQMZkseLyorK5GfV43SXuUo8B6UhxFRp3W0LRFR29iWiLoH2xJR92F7IuoebEt0sPP5fB2et0ejaeXl5ZAkCbW1tSnTa2tr0adPn6zv6dOnT5vz2/+tra1F3759U+YZM2YMADjTR40a5bzu9XoxdOhQ7NixI+tyvV4vvF5vxnRRFA+JE4UgCO1uq2H9V9XNJ1puSYJu4JDYP0Qd1ZG2RETtY1si6h5sS0Tdh+2JqHuwLdHBrDPHdY+2AI/Hg7Fjx2LOnDnONF3XMWfOHJx88slZ33PyySenzA8As2bNcuYfMmQI+vTpkzJPMBjE4sWLnXnGjh0Lr9eL9evXO/MoioJt27Zh0KBB3bZ9hxpNN8OysmpmIrtEAaputPUWIiIiIiIiIiIiStPj/c4nTZqEa665BieccAJOOukkPPHEEwiHw7j22msBAFdffTX69++PKVOmAABuuukmnHHGGXj00Udx/vnn46233sLSpUvxwgsvADCfqNx888148MEHMXz4cAwZMgR33303+vXrh4kTJwIAioqK8Itf/AKTJ0/GwIEDMWjQIDz88MMAgEsvvbSnN/mgZQdgG1plAIBLEqBqDMoSERERERERERF1Ro8HZS+//HLU19fjnnvuQU1NDcaMGYMZM2Y4A3Xt2LEjJbX3lFNOwZtvvom77roLd9xxB4YPH45p06bh6KOPdua57bbbEA6HceONN6KlpQWnnnoqZsyYkVK34eGHH4bL5cJPfvITRKNRjBs3DnPnzkVpaWlPb/JBS9MNHHdYCf5yyTEAALckQtE4YiIREREREREREVFnCIZhMNUxi2AwiOLiYgQCgUNioK+6ujpUVlbmrH0x+I8f4p/Xj8NrC7fhhatPAAA8+ME3+NG3BmFIef7eXF2i/VZH2hIRtY9tiah7sC0RdR+2J6LuwbZEB7vOxBPZAqhddtw+pmhwS4lDxiWJUJkpS0RERERERERE1CkMylK77LG84qoOlyQ40znQFxERERERERERUecxKEvt0pMyZV1icqYsB/oiIiIiIiIiIiLqLAZlqV2abgdldbiTMmXdkghFZ/kCIiIiIiIiIiKizmBQltplDwWXUVNWZKYsERERERERERFRZzEoS+3S7PIFqpZaU5YDfREREREREREREXUag7LULt0wIImCVb4gcci4JQ70RURERERERERE1FkMylK7dN2AWxIQVzW4xESmrCQKUFlTloiIiIiIiIiIqFMYlKV26YY5qFdc0eFKzpQVRSisKUtERERERERERNQpDMpSuzTdgEcSEVM0eFJqynKgLyIiIiIiIiIios5iUJbaZRgG3FZQNjlT1iWJLF9ARERERERERETUSQzKUrt0A3C7zIG+kmvKukVmyhIREREREREREXUWg7LULs2wyheoGtxJmbIc6IuIiIiIiIiIiKjzGJSldum6kTTQV1KmrMSBvoiIDgW6znM9ERERERFRd2JQltqlGwY8rsxMWXOgL2bKEhEd7IbeMR07GiP7ejWIiIiIiIgOGgzKUrt0w8yKjcoa3EmZsi5RhMrsKSKiQ4Ksaft6FYiIiIiIiA4aDMpSuzTdgFsSIKs6XGLikHFLAssXEBEdIkRBaH8mIiIiIiIi6hAGZaldhmHWlI0pqZmykihASxroK6Ywi4qI6GDFoCwREREREVH3YVCW2qUZBrwuEbGMTNnUgb5G3j0DGssZEBEdlDSD53ciIiIiIqLuwqAstUvX4WTKupJrykoCVD11oK/aYGxvrx4REe0FfOhGRERERETUffZKUPbZZ5/F4MGD4fP5MG7cOCxZsqTN+d99912MHDkSPp8PxxxzDKZPn57yumEYuOeee9C3b1/4/X6MHz8eGzduzPpZ8XgcY8aMgSAIWLlyZXdt0iFFTypf4JESh4xLFKGm1ZRtbJX39uoREdFekH6+JyIiIiIioq7r8aDs22+/jUmTJmHy5MlYvnw5Ro8ejQkTJqCuri7r/AsXLsSVV16J6667DitWrMDEiRMxceJErF692pnnoYcewlNPPYWpU6di8eLFyM/Px4QJExCLZWZp3nbbbejXr1+Pbd+hwA7K6gbgdrU90FdjOL63V4+IiPYCZsoSERERERF1nx4Pyj722GO44YYbcO2112LUqFGYOnUq8vLy8PLLL2ed/8knn8S5556LW2+9FUceeSQeeOABHH/88XjmmWcAmFmyTzzxBO666y5cfPHFOPbYY/H666+juroa06ZNS/msjz76CB9//DEeeeSRnt7Mg5qmG/C4zLIF7uRMWUlMGegLAMJxDvZFRHQwYk1ZIiIiIiKi7tOjQVlZlrFs2TKMHz8+sUBRxPjx47Fo0aKs71m0aFHK/AAwYcIEZ/6tW7eipqYmZZ7i4mKMGzcu5TNra2txww034I033kBeXl53btYhRzcSwVh3ck1ZUYCSljkVU3IHZf+zsqpnVpCIiHpc+kM4IiIiIiIi6jpXT354Q0MDNE1D7969U6b37t0b69aty/qempqarPPX1NQ4r9vTcs1jGAZ++tOf4he/+AVOOOEEbNu2rd11jcfjiMcTXe+DwSAAQNd16Af5jaiu6zAMI+d2qprmBGNdApz5RBhQ1MT+EQQgKqs5P+emt1biwmP79sAWEO0f2mtLRAey5PN9T2NbIuoebEtE3Yftiah7sC3Rwa4zx3aPBmX3laeffhqhUAi33357h98zZcoU3HfffRnT6+vrs9aqPZjouo5AIADDMCCKmcnTTU0hKNY+CAaaUSdEAQChmIrWSAR1dXVQdQN5bhENzYGc9YIBoKa2FqIg5Hyd6EDWXlsiOpA1NDWhLl/dK8tiWyLqHmxLRN2H7Ymoe7At0cEuFAp1eN4eDcqWl5dDkiTU1tamTK+trUWfPn2yvqdPnz5tzm//t7a2Fn379k2ZZ8yYMQCAuXPnYtGiRfB6vSmfc8IJJ+BHP/oRXnvttYzl3n777Zg0aZLzdzAYxMCBA1FRUYGioqIObvGBSdd1CIKAioqKrCfFopCE4iIz0t+nsgKVJX4AQIGsQnLXobKyEq1xFSV5Xrh8eaisrMyyDLPMQWmvcnhdUg9uDdG+015bIjqQFRQWo7KyYq8si22JqHuwLRF1H7Ynou7BtkQHO5/P1+F5ezQo6/F4MHbsWMyZMwcTJ04EYDbAOXPm4De/+U3W95x88smYM2cObr75ZmfarFmzcPLJJwMAhgwZgj59+mDOnDlOEDYYDGLx4sX45S9/CQB46qmn8OCDDzrvr66uxoQJE/D2229j3LhxWZfr9XozgriAWQP3UDhRCIKQe1sFAV6XOd3rkpx5PC4XNN18uqVoBkry3AjLGjQjdUAwAIipZnbVproI5q2vw/+cM7xnN4hoH2mzLREdwAzr2N5b2JaIugfbElH3YXsi6h5sS3Qw68xx3ePlCyZNmoRrrrkGJ5xwAk466SQ88cQTCIfDuPbaawEAV199Nfr3748pU6YAAG666SacccYZePTRR3H++efjrbfewtKlS/HCCy8AMBvvzTffjAcffBDDhw/HkCFDcPfdd6Nfv35O4Pewww5LWYeCggIAwLBhwzBgwICe3uSDjqYb8Ljsgb4SB5dbEqBoZgZtTNVRkufG3+ZvRktEwZQfHJPyGeG4OQDYxroQHpu1gUFZIqIDjGEY7c9EREREREREHdLjQdnLL78c9fX1uOeee1BTU4MxY8ZgxowZzkBdO3bsSIkin3LKKXjzzTdx11134Y477sDw4cMxbdo0HH300c48t912G8LhMG688Ua0tLTg1FNPxYwZMzqVIkwdpxuGE4x1uxLflSAIAMz6sDFFQ7HfDQCoCUQzPiMqm0FZVeNNPRHRgYhjMRAREREREXWfvTLQ129+85uc5Qrmz5+fMe3SSy/FpZdemvPzBEHA/fffj/vvv79Dyx88eDAzfPZASlBWyj5IV1zRUez3AABcUmaqdkQxyxc0R+R2l6doOprCMnoXMchORLS/4K8oERERERFR92EBD2qXrgOSaAZj3Rm1MQyE4ypeXrDVyZSVhMzAbcTKlG2OKO0u74OvqzHuz3P2bKWJiKhb6Xy4SURERERE1G0YlKV2aYbhBFpFMTPg2tAax7+W7UoEZbNk00biGjySiJaIjEJv2wnav3v7q25YayIi6k7scUJERERERNR9GJSldhmGgSzJrw57sK98rwQAELPMHFM0FPpcaI7IKPC1HZQ9ZVgvnHp4eddXmIj22PbGMF5dsHVfrwbtR3TGZImIiIiIiLoNg7LULi2pfEEmAXHVDMrq1h27lmU0GEXT4fdIaIkoKGgnUzbPI8Hr4qFJ1FMaWuPtZj0u296Me//vm720RnQgYPkCIiIiIiKi7sPIF7VLN4ys2a822QrKCtY8qpZ5467oBvKsoGx+O0FZIupZJzw4G0u3N7c5T+4HMXSoYqYsERERERFR92FQltqlG0bWWrIA4HEJaI2rqCj04swjKrDtL+dnvXFXNR1+t4SmiIzCdsoXAIAgJDJviaj7ZXt4kqytBzG0/5i2osopIdPTWFOWiIiIiIio+zAoS+3SdAMuUcDWKd/LeM3nNrNff33mMAzqlZ/1/aurAqgPxeH3SIjEVXiktg87wwD8HhdiqtYt609EmQy0HWBjTPbAcPPbK1EfiqdM+9OH30DtgUAtyxcQERERERF1H/Yjp3apulm+QMgSpfG7JQSiCjwuKef7L3j6cxx/WAmK/W4o7WTn2fLcEsJxDXkeHqJEPaKdphhT9k72Je25iKym/L2tMQJVN9DGablLspQLJyIiIiIioi5ipiy1S9eNnPUlE0HZtg8lTTeQ53FB7kD2liAAPreImMJMWaJ9JapocLGu7AEhHE89V2q60SNZrcyUJSIiIiIi6j4MyhIAoCqQezR2zTCQq+JAnscMynrbCMp6XSIaWmX4PWbaligK0NqpF+t1S4izfAFRj2kvvBaTNfjc3ZxqST0iqmQGZds7x3YFY7JERERERETdh0FZAgBc8spqbKprzfqabpUvyMbnkdASkdvMlPV7JIRiCvxWgMfjEiGr2TNmVU2HKAjwuUR2nybqQWo7QbuowqDsgSJ9UETdMHqk1AAzZYmIiIiIiLoPg7LkiOYIgmp7WL7A55LQGldRWehFZaEXXil3UFbWdHhcIjNliTrh4ZnrMG99Xafeo+Rog7aoosHv4U/EgUBJC8pqugHNMBCIKDl7QHRFDyTfEhERERERHbJ4x03Y1RwBgJw1XNUOBGW9ueobAPC6RegGcPaRlVjwx7PhcYmIaxp2B6LY3hhOmVdRDXgkET63xExZog76z8pqfLm1KWN6tjZmU9tJpYwpGnzdPVIU9Qg1rVa3atWUHX3/x1i+o2WPP98wzN8AZsoSERERERF1HwZlCbO+MTPsInL2oKxuGJBylC/weyS0RBR43W1nygKAWxLhlkSnfMGU6evwh/e+Tpk3rmnwuEQO9EXUCblKgjw8cz1ueeerrO9pL+tR1Qy42njYQvsPRUsrX6AbGSUN9oTdW4IhWSIiIiIiou7DO25CWb4HABCKKVlf13S0mSkbjCrwSLkz6uyArT2Su9sqX6DqOlxi6iEoq2b5Ap+LmbJEhmF0qPu5RxKhaJntxS2KGV3bbe0NBNXWAH+0f0nPetYMs3wBgDYHYewo3TDP391ZCoGIiIiIiOhQx1tuwnlH9wEAhGJq1td1w4CYKyjrkdDSTk1ZOyjgtiI8HpcIRTOsTLzUz5VVPal8ATNl6dD20eoaTHx2QbvzeVwi5CxBWZck5Kwd215XdKONDHkClm5rwvefa/+72RvULJmy7QXdO0M3DLhEoVuzb4mIiIiIiA51DMqSE1BtjWcPymp6G+UL3BIistb2QF/WCO52ANbjZMoaTvYsYAYSzMGFJLN8AQf6okNcc0TGqqpAu/OJgpA1COeWxJy1Y9sLymp67ocxBDSFZazohnqt3SE9S1rVDdhfe7YM6s7SdANuSeRAX0RERERERN2IQVlyqG10c85VvsAOuLbVRdYjZWbKypoGVTcwe21ixPihd0xHMKpaQVmWLyDqTCBMQGYblUShjXad+ndEVjNeF5kpm5PXvf8Mgpb+HWt6onxBru+/MzTD7NXAgb6IiIiIiIi6D4Oy5EgfwdvWVlDW7zEDE7kyZQ3DgB3XcVv1Y70uEXFVh9+qNZtcp7AlIsPvluB1caAvIk1PtJ+2tBUqy/X25ABbbTCGUffMzHid5Qty62yt1qaw3CPrIYlCxrlbNxLlC3KVr+gMXTfgEkV0NiYbVzXsbIrs8fKJiIiIiIgORgzKEgBg6qUjctYgNAf8yR6cyWsnKKsbiWw7p3xB0kjxoweWIJ4UNGiOKPC7zUzZOIOydIhrCssoy/N0aN5s8dO2QqrJ9UF3NWcGzszyBR1a9CHJ3YlR0FoiMo5/YFaPrIcrSza0phtO0D3XQG+doRvoUqbs6qogTnto3h4vn4iIiIiI6GDEW24CABT5XDm7ueq6kbMbs9/qwuvJEaBQdd0JxmbUlNUMDK8sQEROBF8DUQU+OyjbDRleRAey95btQkWht/0ZOxksEwSklEVoiSgZ82R7GDNjdc1+M7jVvmZk2ecfr6nB64u2ZUzPNYhid3BLYsZAX1rSQF+5ekB0hmbV/+5sfJeJ1kRERERERLntlaDss88+i8GDB8Pn82HcuHFYsmRJm/O/++67GDlyJHw+H4455hhMnz495XXDMHDPPfegb9++8Pv9GD9+PDZu3Oi8vm3bNlx33XUYMmQI/H4/hg0bhsmTJ0OWe6b76MFAyjFQELBnNWXN94q47IQBTuA2eaT4PI+EiKw6WXvhuAqXJMDnFvH8p1vw5OyNWT+X6FBQXuDFgFJ/m/MErICqIAA1gViHgnCiIDg1RwEzQz1dtocxtcEYvqkOZswblTU091D3/P3FpLdXoi4Uc/7Odr5csKkBby7ekTE9+cFTd3NJApS0wdw0IylTVtuzTNlXFmyFbtgDfXXus/RuyNIlIiIiIiI6WPV4UPbtt9/GpEmTMHnyZCxfvhyjR4/GhAkTUFdXl3X+hQsX4sorr8R1112HFStWYOLEiZg4cSJWr17tzPPQQw/hqaeewtSpU7F48WLk5+djwoQJiMXMG+Z169ZB13U8//zzWLNmDR5//HFMnToVd9xxR09v7gFLEoWc3Vw1w8ySysYOxgppwRsjaZAZlyjgoR+OduZxS6IzIrjfIyGmaE5WbGtchUsUnWDvhtrQHm4Z0YGryO9qd57R93/s/P+3pszBzDW1zt+5QmKSIKQEzFoislOKxJbtYYxhZM+af/Tj9bj+9aXtruuB7P0VVYgmBVe1LAHK9POgLSz3XKasS8zMlNV18x+APR6c677/+8bMlJWErNnBbcn1oI+IiIiIiIj2QlD2scceww033IBrr70Wo0aNwtSpU5GXl4eXX3456/xPPvkkzj33XNx666048sgj8cADD+D444/HM888A8AMCjzxxBO46667cPHFF+PYY4/F66+/jurqakybNg0AcO655+KVV17Bd7/7XQwdOhQXXXQRfv/73+P999/v6c09YLlEAVqOjCqztmT2YEO2IIQkJm7GNS0zsGPXlDUA5LldiMiaM6hXOK7CJQpOULbNophEhzg7Kza55UaTajGrmp619qkopgbrWiIKStNq12Yb6MusEZ25HrsDsYyg7sEoOSap50hIzhYEjfVgpqxbSgz0ZQdNNd1wgsaqbqCqJYqHZ67r8jLiqg6XKHa6fEG2wDURERERERGZejQoK8syli1bhvHjxycWKIoYP348Fi1alPU9ixYtSpkfACZMmODMv3XrVtTU1KTMU1xcjHHjxuX8TAAIBAIoKyvbk805qEmiWf81G03v3CjsLlF06tPambLJPC4R4bgGlyhAEICvdgWcQFJYtsoX2Bm4XdkYooNG2y1ATilVIMAlCoirWtLrhjMIn6YbWF0VcLJdk7MYmyIySvPdmUtPW7xuGFkfxEQVzakvvaf++N7X+HJbU7d8VndLDjJmC76KgpC1vG+uet3dwSxfYODzjQ049a/znOXZ36+uG1i6rQnPztvc5WXEVc2qKdvZ8gVdXiQREREREdFBr/2+sXugoaEBmqahd+/eKdN79+6NdeuyZ+3U1NRknb+mpsZ53Z6Wa550mzZtwtNPP41HHnkk57rG43HE43Hn72DQrJuo6zr0g/zOUtd1iIKZVadpWkbQRdMNCDBy7oej+xelvCYKgKJq8EgCFFWDKCDldbcooDYYQ0meG3PX1WLlzgBO/t1p8LslhOMqJCE1G+9g3/908NB1HYaRu610hqYbZrvUE58XlTWIIuB1mQHQmNUtXjcMwDDgdYmIyZozv6xqcIsCVFXDjuYILnj6c6x/YAI8kgg96XMbW+MozfOkrLcdf0uepuuGOUhY2vaZGZpCt2z3B19XY2hFPsYeVrLHn9XdNC3xe2Bnp6Zus1nLNXnau8t2obzAk2Xe7uESRSiqjrpgFFUtUec3S9U0Zz3tYHJXlx+VzR4Mmm7guXkbccWJA1GSllmdjaZrXV5ud7YlokMZ2xJR92F7IuoebEt0sOvMsd2jQdn9QVVVFc4991xceumluOGGG3LON2XKFNx3330Z0+vr651atQcrXdcRDgXRGoli2J0zMPuXY1DgTWS9hSMRNDU2wIhmZtIBwEuXDk+pEazIcSxauwOPzd+JKRcMhRyPpbweDrWirjkISRTQv9CFlQCqaxtQ6BXR3BpFKBhAXZ0VVEp7L9H+TNd1BAJWNqrY9Y4IoZiKz7cGIOgKNEVH9e5auCQBv5u2EYPLfLjp9IEAgIawOUCXrqoIR6OQRKApEHTaTCgcxZaGVhx170y8fMWRAIDdNXVwi0AgFHLma43EIAhIaWtx2XxIlTwt1BqCYBgZbVKWZeii0C1t1e8WUdsY2C/bfX1DI4oQAQA0NbcASN0/0WgEqqqmTPvDe6vw8EXDAAC1tbU56852lWBoCLaG0diiO+ujajoam5oBAC2BAGTNDPB3dZ/W1DVC0xSEw2E8vXg3RpaKGNUnv933NTYFnHXqrO5qS0SHOrYlou7D9kTUPdiW6GAXCnV8bKQeDcqWl5dDkiTU1tamTK+trUWfPn2yvqdPnz5tzm//t7a2Fn379k2ZZ8yYMSnvq66uxllnnYVTTjkFL7zwQpvrevvtt2PSpEnO38FgEAMHDkRFRQWKiora3tADnK7riCg6XB5z9PSwmIehlcXO6x5vDfr0rkSxP3tQNl1+Xg1aDS/W1UVQUlqGwoJWVFZWOq/3lr1QhCDKi3yYfPbhEP+9Gv7CYpQV+CDrQEWvMlRWlgIAfD5fynsPRp9tbMC3hpZlrf25r+wORNG32L+vV+OAo+s6BEFARUXFHl1gLP6qGvfN3IYrTxqIiKyhoLQMRT43mmIbMNzjd9qE3GwGCP1eD9weLyRRRH5+vvO65K6CrBqIqwY8+eZ5rKSsF/K8bvjz8lHaqxySIMDr3QEBQkpb83p2AkDKNH9eCJIkZrRJt2cHPFmmd4UkiigoyN8v231JaSkqK839WNhgZp8mr2d+XhNEKZyx7gWF5nvKyiu6vZ37PBvg9vrgyyuAKJjro0NAUXGJuU4FhdBjKvI8ri7v07zCIuT5muH35wEACotLnHN0WwqtKhRdWW53tSWiQx3bElH3YXsi6h5sS3Sw8/l8HZ63R4OyHo8HY8eOxZw5czBx4kQAZgOcM2cOfvOb32R9z8knn4w5c+bg5ptvdqbNmjULJ598MgBgyJAh6NOnD+bMmeMEYYPBIBYvXoxf/vKXznuqqqpw1llnYezYsXjllVfabexerxderzdjuiiKh8SJwiWJTv0/RUt9YmWOvN3x/SCJAmKq1V0WAtxp7/V5JLTGVfjcEvK9bkQVHXHVQEmeG7uao/C4JGd+URAO+v1/zStfYu4tZ2BoRcG+XhUAQCCi4Nt/nY9tfzl/X6/KAUmwjllRFPHwzHW45TtH5BwoL5ewbDbGAq8Lug40hRX84b1ViCka/N5E+1CsNitZgzAJggAktRlFM5y6s7I1kJ9mwKkze9nzX+DKkw6DALO+syAITianndCZ3P5UHRntGTAr36a31ZiiwesSO50ZqhnZl7E/MJDYRru6avJ62oMapq+7bpjTNQPwiiIaWuMoL8j8vekKtyQ69WPt70DTDRgwv0MdQFjWUOB1dXmfxlUDbkl0tlkzgFBcg1sSkOfJfRlhl8Do6nKT2xIRdR3bElH3YXsi6h5sS3Qw68xx3eMtYNKkSXjxxRfx2muvYe3atfjlL3+JcDiMa6+9FgBw9dVX4/bbb3fmv+mmmzBjxgw8+uijWLduHe69914sXbrUCeIKgoCbb74ZDz74IP773/9i1apVuPrqq9GvXz8n8FtVVYUzzzwThx12GB555BHU19ejpqYmZ81ZAlyCAMWKysaU1PoXmpEINnSEJAqQrcGGNN3IeK9bEtEaU+F1iRBFAetqgmiNKyjxexCRNWf+q8Ydtl9lj/akzg6g05NSB4+ijgrGFDw5Z2PKtGfnbe7S/rSbjN/jgscloiYYw8w1tVC11IHzFOuzBcEahAtIGWjKMAwoWmLQPfM9BrxuEZoOVLVE0Rg2M+QlMXXwr2ziqgavu2NtcuTdM7B8R0uH5k1mnjP2z3avpwz0lWUGIXtb1gzze5NVHbpu4IQHZ3fbOrkkEYpm7jO7dqxmGNANQBIE6LqBcFxFvrfrA7HFVd0a6Mv8W9UM/Pqfy/GnD9e2+b796bxGRERERES0v+nxmrKXX3456uvrcc8996CmpgZjxozBjBkznIG6duzYkRJFPuWUU/Dmm2/irrvuwh133IHhw4dj2rRpOProo515brvtNoTDYdx4441oaWnBqaeeihkzZjgpwrNmzcKmTZuwadMmDBgwIGV9DN4kZiWJcAIyMUVLeU3XzdHaO0oUBMRUezCc1CASAHgkEaGY6gxWtKs5iiVbm1Hsd6M1rjqB2HsuGIVb3vmqy9t0IAnHtfZn2kvi6v6zLgeS2kAMT87ZhCuPGZsyPa7o8Lm7FhDL90gIx0UEo+aAXmbgNPG6bLUzO6DaVjPVrIcuiqbD65KgG0bK+1yiCFU34GpjVWOKDl+WGXKdVYMxpc3ty2Z/PkcnB2KzBbBFQcCOpgg+3VCP00dUJM2rw++WIGt6tz/0cIkCVF2HICQPzmaY523r+w3LKgq8Xf+5j6s6XJLgfDeKrkPTDQRjapvv4/MdIiIiIiKi3PZKOtJvfvMbbN++HfF4HIsXL8a4ceOc1+bPn49XX301Zf5LL70U69evRzwex+rVq/G9730v5XVBEHD//fejpqYGsVgMs2fPxogRI5zXf/rTn8IwjKz/KLvkLsZxNS1TNku2a1tcooCInDtT1usSEYgq8CVl3LVEZZTkuSFbN//2fIdKgLA13nZwo7vsbIrgs431bc6T/v1Tx+RqI3Et+zF87StLcn6WHbjL85qZsiEruOkSBSe4CqQGZc3Tm5kZabObtdclOgEyVTfgdYlmUFYQEFc1iIIASRScbNpcYorWqQBzKC1o9+jH69s9D7skEeo+jubN+qYWn27IbCfJgdhsWaB2pvKqqkDKdEU14PNIKeUk9tSkt1cCMHseqJoBTTecDGvNMBBXdXglEZoBBGMqCnxtB2WbwzLOfeJT5+/rX/vS+a5iiga3JDrbrGoGfG4RcaXt87PG31wiIiIiIqKc9s8+orRPJWfKaroBRdMhdSZTVhQQUzQngysjU9Zl1lQsSho4rKo5iuI882+3lTnd3aOU78/2VlD2naU7cdu/vm5znvRMaQBYsKkBdaFYT63WQcEuE5AunlYOJK5qaI2rmLc+d3DcDrbmeyR4XaKTkZgrU1YQEhmv2daiwOtKy5Q1645KkoBwXIPXLcElCdBybIMtKmvwezoWlBUEoDUtKPv03E3OA5t0n26ox9rdQXgk0SnLsK+8vmgb3vpyR8b05EBstkxZ+5Qlq5nfud8tQVF1KGmvXTZ1EdZUJ4K4UVnD3HWpg11m8/6KKjSHZbgkAYpmZq66RBG6bsAwzMC+x2X+HZU1+N1tB2UVTce6msQoobPX1jnbmFm+wMy2jrXzAIcPQomIiIiIiHJjUJYyxJKyU99btgtz1tV1aqAiSRCczKps9SE9LrObdLEVlH3k0tHY1RxFid9jvl86dIKxdgA0vJeCsnagpi3ZMmV/9NJiLN/e3FOrtc/8e8WulMzSPZEeiLOl78/fv/s1fmdlOeZivyfPI8HjEhGMmpmykvWgw5nPCl6KVi1TUUitKWtLzoJNlC9ItFWPJMIlJupKmzLbYVhWkZclKJu8zA21Iby6YCsKvS4nwzdZevas7U8frsXbX+40g4zd9J10lShkr6+bfKxkDcpa+0zV04OyOnxuM9icnim7ZFsTPlpVg4dnrgNg1vn92atLO7Segahi1o01DKdXgp2dKqtmW8+VrdoSkbE7EHX+zpYlbb83rmrmYJBO+QIzoK/pbQdl26tRTEREREREdChjUJZS+NxiykBft73XdlZlNpIoICprTvA1W01ZAOiVb44+flS/IlS1RFHiZMomz39wB2h3NUcxonfBXgvKukWzq3Nb7ECxph/8JT9+9/ZXaIrIHZ7/7mmrsd7KJgxEUwOOco4yBemZxzuaImjvGUciKOuCRxKd2qyiIKQEQJMDwbphZkpquo76UByXTV0EAHjjupNweGWBEyBTrYG+dMOAATNQ73WLcFkPUVIJGYFIV45BuOws0U/W1+OvM9ajyO/GmupgxnzZArX2+w1rQKx9Xb7ADHJnTk8OcGZrGvY+SG9jcTVRU1ZRM9+4Ymcznv9kCwIRJSOg2xZF051lOkFZPRGU9bpEROIq/FlKTkz9ZAuueOGLxLZlC8rambKKmSlrb7Oq6RnHYrKl25pQH4ozKEsd0l0PxoiIiIiIDjQMylIKn1va4zquoiggqphB2Ww1ZV1WULZfiTkwW3mBGZwtsTJn7dcPBcGYgv4lfrR280Bf89bVZc3cVHQd7nYyke2A4EufbcH/vLXSmR5T9m2grKfkynDN5l/LduGzjfWoDcYw+r6P0z4ne2BB0XRc/MznTjBS0/V2B0Cyu+/neyV43RKaw2bgWBRTu9Db624HvyTRzDLdWBfCkm1NiMgaThtegTyPywkU2uUL7G7uimY4mbLp2ZKSmL12alsislm6pNDnxn+/qs76ejaiYHaPd0tizlIQPWF3IIrqlmjKNEkUsgaKkuOlbdVLTc+GjStm2YeYkn2gL8MwM1VH3/9xuw9Nsi3HMMz1kUTB+b7iqnkOjmuJOt3Jygs8qA/Fnb+TS0bY2+4M/qhqcElCSk1ZAZnHxhdbGrGpLoTf/u8KvLJga6ePHTr0BCIKbn9/1R5/zuqqACLy3nm4SURERETUXQ6d6Be1yzAAn0va4+Cb2SXazAyLKVrWgAAAp3xBWb5ZtsCuMduZQcWSTXx2QZfety/FFR1l+d4uZcq2lcV67atfYmNdKGN6VNaQ52m7tqSdFReOq5i+ajcAMwMw2s6gPgeqzgxs5rKCXtl2vR0gq2uV8a9lu5zvR9UNfLUrgDorAKbrmd3b09l1R/1uF7ySiMawDFFIBC6d+TQdopAInrkkM1PSrmPbEklkpdpBREUz4HVJ0AwDumFAsTIqpSwZqqIgpAUfs7fN5DliqgZRFDCidwFG9inM3LYcEWk74OyShE5li+6pyf9Zgzv+nRoUEgUha0AxvaZs8rnq/eW7UoKWyWKqjkKfG1FZy/oQIFut2jcWbcMbX2xvc92TPys5U9YtCU75AlnVs55T7QHC0pcLwCljkZwpmzzQl5ajVMbk/6zBPxfvQL7XhZjS/sMHouaIjJnf1OCDrzMf4HTGBU9/jk83NHTTWhERERER7R0MylKKbCNq//WSYzr1GWamrIqSPDdCMTVnkNXOiLVft4O07WVyZhNXNazc2dLp9+1rcVVDWb67SwN9HXHXDCeDMpvkLDhbc0Rx9nNb61SS50EgqsBr1Z/1uyVEc2Q4HqjsbMBOZYZbh2a2wKIdINtQF8Ft761yAlr2vHbgXTeMdrNzVd3A6AHF6FPsg8cloiksI9/rgiAIKcF4WdXNLFhrWW7RrFsaUzRUFnqdsgdAIsCm6mawzjDgDAjldYlmkC4tO1QUBaTGR3M/CLC70RuG2b3d55IwoNSfuZ9yBWWtQKgkCHs1mOfOUrbBDEZnzpscoLbX1Tbpna+c9ba/c7t9yqqOIp8bEVnNmSlrs7+DL7Y2YcZq86HIta8sybrudkaxYAXm7Vq4bskMxtqDpnVkoEYlS4BW0cxyEnFVcwb6sjOqhTbKF+i6AUlMZO9S+1oi8j4v27EvhGIqWiIKnp6zaY8/a1VVy56vEBERERHRXsSgLDkEAfC6pJTMwTEDS/D94wZ06nNcVk3ZkjwPQjElo6ZsNtecPAi9CjzW+zt/WOYaPGh/F1fNTNmuBGVlTce2xnDO19NrngJmYDDfm1lfMmWdFB0leW7UheIo8JpZtX63lJEp29RGQPhAYGcDxjuRGS7ADKBly661A632PrODa3Y2YjipREVy2YE3vtiOKR+tTf0sTccLV5+AsnwPvC4RzWEZBV5XSlasPZ/fIznBHEkUzKCsqqGyyJvSLuz6znb5guTaox4nUzatfEFGpmzHJAaUy2z7uUoTiIIZABbTSgfUh+L404ffdHodOsreZ+nTsmWip9fXTT9V2d+rvY3/89YKAObxUuR3ISJrGQ+97OXZ7O/S3h8AMG99fdYHAcnT7O9X0w0zQ9Ya0C05U3bo7R+iJhBL2bal25rw/ecWOMeDYRhOhnVc1cySNooOlyTCMAzn+xEEwICBSe+szHiwoSXNJwnZ9yWlOuUvczFzTe2+Xo29asbqGsxbXwfAHERwT3hdIp6dt7k7VouIiIiIaK9hUJZSmAN9JW6wywu8VnCl4yTRLF9Q4ncjGFMhdSDIet/FRzvd6jsSxE1nB58OtAFD4qqOXgWedssXnPnwPASSuqJvbTCDsdkCo3YAJFs2ZrYux5nrpKEsz4O6UByFPvM78VmlKJId/8CslEzMA43TNbsT5QsEQYCBHJmyWmKANCApKGtF1ux6hx6X6AThZVXHkq1N+HpnIOWzFFV32oHHJaIpYmbKusXUrE4zU1ZyprklAf/4Yge+3NaM0jxPSo1FTdMhigIUzYDPbZYvcEsCwrIKj8uuKaunHFOSKODlz7fi3v+uaXu/IFFvVRCsOrXWecM+HpODwNmIghnMc4mpgeDV1QG8+NnWNpe/J+ySDymE7LV0k2fTDWRkoMas4KT9nTclZcoW+twI58iUdSfV0bbXJf0sGEx6yGKf55I/S7fLF1gDvtnBdkVLZKvqBvCtKXPw5bZm83sRzON0xY4W5yHFzDW1OOUvc83tUXSz94R1POq6FajXDQgwM2ffX17lnJvs3aEbhpP5bNaizdhkShORtS71EjmQvbxgK56Za2bIpv++dNaph5fjnJGV3bFaRERERER7DYOylMKbEXzr/N20KAiIKRrK8j1oDsudDrKKXQjKtlpB2QOl7unCTWbtu7iiodDrylln07atMYK6UMz5e2dTBCcOLs2aDavaNSVzfGZ7PZljTqZsDAU+s9SB35O9fMGe3kjvS3Y2Y2fKFwhWUDtbYDGRJWn+1854tJdjB2Ldkug8RJBVHYZhZh3+bf5m5ztWNB1uK6jpcYmIKToKvC64JHMgL1tcNWs320FMuyTI9sYwCryulKxUVTfgdZkPXczyBWawLhLX4HVJcInma8c/MMt5jygI+GRDPXY2RdrMdpTEzIxaM/sWGWUcch3r9iBVdhd8Zxt7+BiThMwBzmCY254ueb10q1xAMjvr2t5Gu33Kmo4inwuRuJY1Mzv5HJm8LkbS+TeYlvUMpB6HqhWU1XXAIwmIt1FTtiksw4AZ+LWD5/a2bWlodeaLKRo8khmUlUQBBoyk7ykRuA6m9VTQdThZ3dnKQ1B2h9puiimacwx5umGAzw5U6SAiog7SdIM9XYiI9gIGZcmhGwbyPN0w0JdoBkfLCzxojihZawpu+8v5Wd/76KWj06Z07ILAHtm+KwNm7W3NYRlXvbQYhmEgrurwubOXE5j0zkpUtUSd7W9MymAMxVQMKM3LGpS1s/Laq1uaS1zVUJrnQV0wjsKk8gWRLMGxmHzg1kBMHsSoPSt3tphdtmHVhM1RU9YMhicyZe2anoCZCQeYwQc7QOt0D1d1/HXGOizd1gwAUKz3AmZJEQBWUFZ0BgGzl5nnkZyyA3Zwrzms4Ig+hbjx9KEAzAzamKI5Gc92+QK3JCKiWJmykpBRh1gUzMCdZh2r3hzHanIGtv1fjySmBGtlTXfKJ2Rj1yi1B6uy7en5qD12dm6ybFmy6dPtLvrJ7ExZOxhuZ5DKqmbVlNWcY+fZeZuwdFuT81nO51rbnp7VLqs6Xvh0MxZtbnTmSd5Pmq47A3y5nfIFiYBqsqiSlEGdFDTP90h4aMZ657W4qjmlENxWxmvy92Ovn50xb/9t19vVrWzsXPvzQNbeA6naYKzTN7Md/f06WG6So7LmPGAotB4AarrR6frldjkNIiLqPr97eyWG3D59X68GEdFBj0FZcmi6AZ9Lajdrsz2iICCqaOhV4EUg2rlM2UvGptavdYmZgw9lE4qrKPK5EN7Hg1FtbQi3Wx/W7i7fGlfNQJc16FK695dXYX1N0Pm85PqgoZiCAaX+NoKyLsRVHT/NMUBQXNVyBhViio6SfDd0w4Df6hrvd0uIWft29jeJuocHSmZyNmpS0Ko9E59dgMaw7IxYny3gbX6XiZrMqmbA5zbnL/C6nICL2ypfkNwt3M6qbY6YgXdF1Z0u7XYmo9clwi2mZnXKmjnQlx0kc1ndnwNRBWePrMQd3zvSmi46XdGjspkZawfYInEzG9IlCk4tUZskClbJAx0RWUNejqBsMifzzSVCSiq3oKhm/dtc+9sOeKdn3XZqILYu0LJkvBqGGSROp6cFT9ODnXZWr6Lp0HUDISf4bteUVZ3t+ccX2/HJhnq4RCHleLLPv+kDacmqjr9/vhVfbGl0yiOkBmWBPI8LUTu7VTGPx2wDfQlJxRHsgL6mG6gs8qXMF1OsEgiqDpcoOt+PqhspwWz7QYGim4OLGQYAa8A2tyR2KSi7eEsj3l1ZlzH9H19s7/Rndbe1u4M4avLMNucZ9+c5WNGJwSc9ktjhuqpDbp9+QPdSqAnEoGiJB5JFPhcKrFI5S7c14aXPtnTq81qiCkryPN2+nkREB5NARMGKHc2IyhoWbm7ApHdW4ostjTnnb2g1H9Qrmo4djZG9tZpE3cauW0+0v2NQlhyabkBKqmn3+cYGzF7b+ZOZJAqIKzpK83JnynaU22UGtQzDcOqoZhOIKuhb7N/nmbLfffwT/GvpzjbnsQMwLREFcVWD1y3CLYlZb7LrgnHUh+LoXeRFRFaxbHsTFm1utDJlswdl41ZQVlZ1zF9fn3Udnpy9EVf/fYk1f+py7UzZAq8LfrdkBhA9EuJWsOj615eiNmh2sz+gg7K6GSxtL+hnfy91oRjcVuarnfWanLEmazoKvJLzYECxBuFSNDOb1R7oy21lGhb63JA13clg9bsTWep28AtIdOsVhMzuubJqBlqdoKxVv7kpLKdkYLtFARFZswZsM4PRum7AZQWCvG4zq7XZyey0BpsSBUiCWYfWHCTOBVnTcee/V+XcX/Z7PZIIKWlgMt0waxPnzpQ1/ysKArSksgsxRe+Wrs25aLrhBLNtBowsQ5S1X77Abg+qZjg1WgHzWCjwWjVlrf1T6HMhFFPN82VSUDa5pmxy+QJZ06AbiQHAvC4xLUhsHm9RRYNbEp3B1rKVLwDswHNSDWTNQHlBamArKmtWXVozU9Zwlm9A1RL7LbmcgtvKkNZ1w8qU7Vr5gjnr6vGvrzLPX3dNW93lXgB7ym7vNYFYxjY9PHMd3v5yR8q0TbWt6Kh8r9SpASvrgvH2Z9pPffuvc/HB19Xwe8xz1LDKAmeAxIisoS7UuW3b2hBGrwIPVu4MoC4Ya/8NbWiNq3i3nd9wIqL92btLd+LJ2RvxTXUQN1sDjgLAkm1N+P5zC3H2o/Nx1YuL8f7yKuxoyh5sfemzLagJxCCJAobf+REmPPFph5a9uI0gL9HepGg6rn3ly329Gj3uQBtPh7JjUJYcmjVSth1nWLGjuUufY48AXuRzmTVl92DwErs7blTRcNYj83PO1xKR0b/U7wTL9oU5a2uhaJnZZulkTYcoAGuqA0422+DyfGxPegodlTWUF3gRjCmoD8UxuFc+WuMqHp+1EW98sc3KlM1dvqDA68JO60LLDiRMePxTbGuMQBTMAF299QT8iLtmOEFWwAzqlua5UeBzIc8jIRhVUOh1pXTzb2yVnfU8UMQULSWQomoG8ry5MzdtdqZya0x1AlT2AFrJ75VVHfleFyJKIovR55agWv9ND2AX+sx9GpE15Hkk5HslROJqxsMHrztxmja/ysQ2KFbgzWZnLkYVDT5XUlBWEs1p1nrYA325rEH57ExZuxu63Q1fEgSIoplVbAePw3EV/1ycGnxKZnfPtzNl7dikbpiZw0o7+zt9oC/7wUVPyZYpawY/M+dNTvjUDViDXyV9H2qiFED6gGx+jwhFNZzseL9bQkRWMzJlnUzotPIFcav+sCiYA4nZg3g522GVn4nIqlm+QDXLVCiaHZRNzfK1SwzYwWNVN/D94wZgyZ3nYHCvPADmceB1SZA13fwuDcOsE2wYTvkLe3vtz5VEwap9rFvzCNC7EEOVVR2eHL8d++Lh28LNDc5NqT2AW/J3/8WWJizZmvjNLC/woqol6szXXg+KAp+rU9sVUfb/Uj25aLqB2mAc/Uv8AICXrzkRHpeIzfWtaI2rTnZWR13yt4Uoz/fimpMHYWZST46uWL69Gbf+6+s9+oz2NLTG0djJbeysa19Z4pR1IjpQHEjXlPuzzfVhvLd8F9buDmLaympnumb9GO8OxNCnyIfSPHdG2SrbE7M3YktD2LmWOWZAMQDgomc+b3PZl7/wRXdsApFjdyDqXE91Rmvag+64quGrTvRgOlAMvWN6u+dOVdMx5v6P99IaUVcwKEuO9O64RX53lz5HtIIqBT4X6lvjKcGhznKLIv7nrRV4cs7GlOlmPdbECag5oqB/ib/D3T+7m6zqeOTjDfBIiXILry7Yin8u3p6RASurOq446TBMX1XjBGgG98pLCcaFYgr6l/rRElGwqiqAoRX5CEZV1ARjqCz0IRhT0a/EnzWzKq5qKPS58e6yXQASNTnX14bQElHgdUuoDcZQmpf4fmsCiaBsTNFQkudBodcNv0dCc0SG1yUiOajTGI478yav884cT9z3B7/8xzL85aO1zt+qriPf42p3ICn7h86uq2lmjWooyXNnD8pa2YWqbg7CpWjmf+OqlpJZa2fK2lmxeR6z/MZZj8xPCejZWaKikDmYlmx1K7drmyZ3ufclBTJdkoC4Fai1B/qyyxcA5gB/Lkl0ttX8Xs31Eq2BsOxjVc2R6Wqzjwl7oC+7q71u2OVRcj/RtcsGpGQgq3shUzYtAmtnhCb+Tq3hGpFVfLqxHm5X6ndib6ui6SllJuza0YquWwFaCZIVjJXSgrKJTFkhZZhFWdXNYLH10Mv+Luz11HS7JrjmDPSVHJRNfpCuWgFTSUxkJds1aSsLfc53ZAfsZVWHSxKsfWVmWSeCvYkSCKJgrrPL+lxnoK8ulC+IqxrcOYKy7QU4e0J1SwwbrMzX5oiMwb3ysL42hEWbG3H3tNVYtr0ZTeHEze1R/YqwpjoIAHh14Tb89OXspWRs+Z5OBmUP8ODF7pYo+hT7cNxhJSjJc8PnlnDOo59gXU0QuwMxnPXIfGyqC3X484rz3Bg7qHSPBwZsjavwukRsqmvF9FW78emGeqzY0YwHPvgGG2vbX5/0h3/Z/P7dr3D7+7l7G3TW2t1BvGf93gPAruYI5q2vx+/f/QprqgPdtpz9weqqQEoJpS+2NPZ4iRvqedNWVOHfK3bhuAcYNOgs+5osuZdAkd+FYExBjXWt3xSW0dAad3psjT+yEkf1K0JloS8lKPvUnI1oaI0jpmgY0bsA44+sdF4rtHqWfb0r9zklUW/eQEtEzjkfUWf8efo63PruV87fCzc1YJb1O7CuJpgzU9S+R95cb167fVMdxMXPLujxh6L7QnulvbY1RjJK1NH+hUFZcqQHZXsVePDYZekDb7VPEhLds0MxdY+y3NwuAQs2NWR01fzg690459FPnL9bYyr6FPsQie+bi/OorKGxNY5eBR6n9urGulY8N28zRt49I2VeWdPRu9CHcNysL+l1SehX4sfuQOIpYEzR0bvQi3+vqMKDH67FiYPLsK0hDFnV4XWLCMVUlPjdGQO+LNve7JQvAICj+xelZMuIAuBziagLxZ0afIIAfGkNOAQAwaiK3kU+FPhcKPabT9GTszEBOBdxydmfz3+yBVfsx0/IDSAlG1nVDeR7Xe1myppBajfiihkAVTQd4biKsjxPys2grJpB3ohsZvgpmpUhK2so8LkQU3QoWiK7sMgqMWESnMxJANiSFKC326QdFEtmB8uyZXXaA+cASZmyHsmqKWsG1uygl50pG1fNDEwzK9oMyApW+QL7NbmNoCqQGDgtfaAvO3M42yBp5h7IHoBTs9Ru7U7ZasMaRurAQfZut7dlc10Yy7Y3Z3TNtwOxqtW9H4ATdDVLN5j70W8FwRXNLCGRfBwlasoiIzitW+uVPIhcclDWPN50J6PbkzTQV3KQWNEMqFb5Cnu6klSOQLMGToopZpayPYidnfmqJmXamtubus52LXDDKl/QlZqyimZknHds++LhW2NrHGX5ZptvCss4un8xznvyM1z54hd4w7oYbmiVsb4mhH98sd2qFW5ud3u1xg3DLKXSmvb7tas58yHXi5+a9Va7klGWfuMy+I8fdvozuktrXEOB14V//+rbEAQBfus64dl5m7FyZwu2NoSxszl3Zoys6gjFFGcfDyjxw+tuv+dDW6Kyhl/9czm+NbQXtjWE8Y8vtuPdZbvw/ecW4u+fb8XKDmTYjLx7Bu75z+o258l2ztkTl05dhFuSblhP/es8AMDMNbWYtqKq25azP5ixugZ3JJXPmfrJZmxNyuajA9PNb6/EO1/ucn43p0xfi4/X1Ozjtdr/balvxeF3foRARMFJf56T8lpLRMEnG+oxsCwPk95ZiSnT1yGiaNb1ogBZ09GvxIe6UCKY+9XOFlS3RDHy7hkQBQEvXXMibj9vJADz3qW9Ejv2PUFDq4wx98/q1m2lQ1eB1yylZ//ef7M7iK93tQAAzn3iM2cgbFnVcfgdiYHpvtrVgoFlfpzz6Cf4bGM9djVH0b/E36Ws2/2RYRj471fVGNQrz+n9mktyO6f9E4Oy5NCSbrKBzK7RHWVn7dmBQe+eZMpaN/TpN7S1wZjThRQwb9IrCrz7LFM2LKtoDMsoy/c4FyW98j3OiT85uJK8X+NWkNXOsATMro0N4TiGVORjt5XBOqJ3IVqiMvK9LkRlDaGYggKfK2MQtEv+thCrqgIosgJyA0ryEEy6iJI1c3kxRTNrVhoGeuV7Um7wVV1Hkc+FQq8LJX436rIEZRusAEVUTgwY1tPBsz1lJGWGPjN3I1StY0HZiKyh2G9mtbqtYFdYVs0ATVJJB7umbMQaSMseIC0sa1apAg0xVXMyWM3B2Mx953WZAcyG1jgkUUgJHvvdEgaW+ZMGzRKs7TEDfG5RTMnq/KN1AW3XawRglSnQ4HOZwVm7Hqldg9ZjLT+mmIES+zuVRDOQs7Mpgta46gQCgdw1jOxtsj/T7rpuGGb2bsfqgSa2R+/h4yrbcZuo6mqtg1XqwW7HyfV+00tiAHbQ09xOn1XX1eeWoGp6IihrBcFzZcqmZ+sqmuHUsdV0wylfYH2FUK1M2aiiwSWKToaxPdBXcoazapVXMLOgE6UH7OOhJhiDYcDKujXr09oPJDyuROarvXpKUqasphtONq9mmIHertS7aqt8QXK721ta4yoOK8tDS0RBc0RBnyxlasryPfjvV1V48MNvnIGrgMQ5f966Oie7I3nQlLiqo1eBB63x1CyGU/86LyPY9KfpZrZ/VzJlh96xf4yi7XOLaInI8CfVvfZnGUSwoY3asi9+tgU/eG4hVN3A+CN745TDy6263F1/MLupzsymGdwrD9e/vtSss510nZFe5iSdvez2AubJNcM7Y0NtCA/PXJcx/YwjKuB1iSkZpLblO1o6vZz9mSgmSmwZhoG1u4PY2RTFsDumt9uLg/ade/+7Jmc3ecD8LV20pREjehcCMBMvDrZjtyfYvwPpARf7mmLJ1ia0xlTMX1+PcFxFa0yFbgDFfjfWVAcxqFe+83u6rSGMulDc+Z7sBIKfnzEM2/5yPgaU5mFzXdt10iPWvdqpf53bfRtJhzy75NaQ26fj+te+xIMfrsXOpgiqrXts+ze3OSJD1Q0s39GMX/1zGX77vyvQv8SPoRX5+Mnfl+DdZbtwdP8iXPTMAixNSkY6UA25fTr+539XYHtjpN2yT22df2n/wKAsOdS0gb7sQVs6yw7s2kFZX5abrY7ySGY36/S6MHbdWltU1lBR6N3jWoNXvdi1TM+IrELTjZSgrCtp3yUHlWUr41AQBMSsrMV8r+Ss+7+W7cLHa2qR73Hhk1vPBAAM6pXndFmWVd3pFty/xI/PNiYGw5FEAfPW1Tn7fkCpPyVTNmoFDOOqjpaogpU7W+B1SU4NUV030BJR4PdIKPC5UJrvQW0wltF9vD4UR0meG1FFw8i7Z2B3IIr56+tQmu9BIKKk3MjuLxRNd76TWWvrrMGXpHa7PkYVDSV+N+Kq2S1ctgb6MrPmstWUTQyO5PdIiMoqCn1uxFQNcSUx4nih1+1cONvTmsKyU8/TVprvwWe3nQ1JQEoQ/vlPt+DDVbshSUJK2YKffXtIxja4XYmasmb2o5TSbd/rMgebi6t2UNZcL8kK7rXGVSzc3OBsFwBE2gl+eFyiGQxMCvp5XbkH+kqVWiu1vWDInkgORjpLT8uUtQcDs1c9OfCc3DU/+Zxk/7+dveezgqi7miPwua3yBZruBGdtdpBY1c2Arf05sqY5e0V1grK6Ve7BbLt+t3m8ua2SA163HbgVUrJVVd1wvn/78xVrXWwjehc4pS7sgb5kK3s1Pchqf8eiAKsshl13FntYviD1e7HrnO9JNmRXtcbNwRWbwjKawzJ+e/Zw/OKMYc7rI/sUwi0JaAor8LslFFoDVxmGgVDMrB38h/e+xt8/NzNdT394nnMcheMqiv3urNl+uR407qvBzrqDz22WxUl+cJT+Xf/xvJFt/p7Xh+LYXN+KmKIh3ytZn5t9wMyOemL2Bjx5xRhMOLoPALNnxeebGpzXY2m/Fenf1y/+sQwAnF4ogFkL79l5m8zX31iGXc0RNIWVDj/wnruu1vmuv9zWhOfmb854/cOvdyOu6vhXUgmDA8W6miCWbe/4+AUxRUNM0bFsezOWbW9GbTCOG15fCqD936S2pPc6ou4zf30dXl24LWf5D0XTMbDMrC/dt9h82LU/P+DfGxpa4yndtXOx7y2emL3R+u8GGIaR8vvw9FXHYWCZH7Km468zzIc6I/sUoiTPjQtH94NmGKgLxXDmI/OxqiqAaisZ5MLR/VKWNaQ8Dx+tbjt72U4CGd67AN8d1RvPzd+EMfd/zIGIKCvDMNBkDXqdSziuoiYYx+4W87i0ByBvaJWxusospdEaV3H7+1/jC2uQuR+9uBgLNpn/73FJuOLEgQCApdua8O3DywHAya49kLT1OxWMth3/sIOy89Z1fgB32jsYlKUEAykdiOU2uo+2xalT6UrcKHWVfaOWfmOqG0ipt6hoOorzzHIJe3JxvXBzY5e6wdl1mnrle5wn18kXRXaXn5+9+iWqWqLwSALyvXa9VimlnmBtMIbqlih8bhGDeuVj/u/PdLqie63uyLZbvnsEfvL3JdB1AzFFQ698D0Ix1QnKHl5Z4AziVeh14btH9YHPbQbfLh7TD/PW1Vk3suZnxlQNfUv8qCz04ZdnDkNxWqasphso9rvR0CqjNM/j3AA3tppZvL0LvfjT9G9w3Wv732iXqmbAbR2bwaiCqKJZNWXbDm5EFQ3FeR7nIYVZU9bMlE0OANhB2bCsO128zZIEGvI9EmTVHCzL60pkytrHiN8jwoDZ3ayyMPtAcZKYGgyzjzMBZhDMZLbZObeckfJetyggqujwucVE+QLDcIK5XidT1hwkLiKrEAXzc+2LpV1NUbNWrpUVGckRMLGbn8dl1rq1M2U1a6CvzgaTNB17NFggAIy466M2Pt9wsvttiXxkk24YcIuJ4KLdBjMG+tIT0+395HMlMk0BA+8vr0KeR7IG+NKyZspK1vvdkojTH5pnlo2wa8RaD2XsALckmANrxVUdfo9d09gqRWHXgxWFlEHDFKveq5S0nppVzsD2s28PQWNYdpYtieayzSxwaz8Zic8DzCw2w+pxoVuZDenZxB0VU3V4Xanfy/efWwgg9dz6t7QgVXeLKWYt6HBcxcCyPDSHZai6+Xszql+RM9+Mm08HADSHZYiCgGK/G3leF/48fS2CMQX5XhfqQvGUsiJ2WR5zsD8XDMPMVkqWKzDZ1Tqa6b+P3RGMWrCpAWc/Oh/ndnB0bq9LREtEQV5SUFYQgKEV+QCAj393Ok4cXIZwGxmnry7cBt0ArnttqXNO9XskvPjZ1qzH2xtfbG/zOPzXsl1YuLkRF43uh0Kv+R3Zo5KfOLgUQKI+u23YHdNTvp/5680HpHbw1jAM3Pru13h45noAwIw1NTj1r/Owdneww+fBn726FFsazOy0lojiBPttn25IBI3dWa7XsmUg70/eWLS93XIPyaKyhkBUwQ+nLsQPpy7CrROOcF7L1lbs47utLCFNNzOwusuG2hB++LeF3fZ5+6tl25uch6wxRcPCpAcYyf7xhTkwaJ31HaypDmDSOyud15+asxHHDijB01ceZ5X+0Z2aqJE96P3WdAAGXWwba1udcSHaYtdt/XDVbgBmcPbP09dCVnXcOuEIvPLTE3FUv2J8dtvZmGsFY2479whcftJAzJl0BsYOKsWPxg3CSX9KlD7YVBvC94/rj6vGHZayrEG98vHqwm0AcveUsgcfLvK5UeR34+M1tWiJKM659EAVjqsdTjbhA56Om7uuDt974Wv84b1VWLUr4DzATHbU5Jn4dEM9GsMyfn7GUGf655sacOMby9C/xI9AVMH/LtmJTzaYv8FRRXOOxeMPK8GNpw/DrROOwIXH9nMGGN2Tc8u+MuT26Rk9h08bXo7HLhvd7nFnn39f/GyLM6299wSiCjNs9yIGZSkhLe7R1QF20gMcBWk3EZ1hB2MyR1DUU27oDMMcKOWxWRvw98+3dnl5QNdO1PbNQK8CrxOos2+a+xb7nJPo3HV1qA3E4HGJKPC6nKCHGcyzuyHFsb0pgnxrvw0uz7c+T0eRlbFpK8v34FtDy1AXiqM2GMNhZXkIxhQUWDeVh1cWYHO9eZN/0pAyPH3lcVi+vQU7m6I4Y0QFAlHF6a7++KwNqG6JId8ahGhknyKU5nlQF0wEZcOyir7FPtQEzMEDgtaPXiimojTPY2ULe9utbdNVbf2AdCjbxjo0g1EF4bjasZqysp0pawVlVbOmbGm+J6U+alzTkZ9UvkDVDHitIGiiXIWWyJT1JQYK87slyKqGsKyissibdT3MQbMS229fFOtGZj3WYRUFKX+7JBEx2Vx2VMnMQPS47JqyZqZsWFbhkgRIghnc+8Fx/dHQGne6sgNoM2ACmIEXV1JNWcMwa552NstRszJGu3qhm541ki69ditg7tPk7FzdMM9FRlpQ1uPKLF/glgSISTVcnaCmZA62dfLQXjisLM/JiHWJAuJJdWTtjFV7IK2qligKvC7nfBy3grJO+QLBfAAWljWnfIEdjPXYgVsxddAw1RqILDkTV0kb8KxfiR9VzVEnO98lCU7GrJZUQ9bebsBsXrqRyMy1a9BqutHpTBlV09ESVTHpncxsoeRz4F9nrOvQsbGxNtSlbJ1L/rYQb3+5E+G4hsPK8tCUNHhJSZbBMBUrk77I70ahz4UXP9uKumAcj102BvN/fyYMIxFUth+YRazvrjWu4vdp2VHJv33Jx1pXM2XT21+uGs+5aLqRMUjG2t1BbKkPY11NxwbmcjJl3clBWcF50uqRUnuPJFtdFcCG2hBOGlyG8gIPlmxtch4Al+WbGaoNrfGMjPy7p61Oqdtus2847v3vGhxWlgdBEFLq4I8/shJ9is2buGxZuLPXJgYbAYCvJn8XMVlDc1jGhtpWJ9M2eTDN9M9KLmWxviaUUUuzJWLWzt3VHMVhaT0p7Cx1wDzGH/jgGxT6XLjhNLPHhLcLD9b3ptI8T6cGH7EfRtpNvrzAg5vOGQ4gcR027s+znfbxk78vwbQVVTjxT7NTeg0l6+6yV1sbwljaiezfA0Fy/WbbJX9b5AxQ+/WuAK56aXHG+xRNd9rI2t1BbKoL4ak5G/H+8ioM/uOH+MnfF+PpuZsgCGZm5icb6nHNK0tQ4vdgS30rRt0zM2u7zeVHL33hLPf4B2YdsAEye78GIgqCOY5bAFiRpcTDi59tRUzV8P3j+uOskYmBun5+xlAcVpaHX515OIp8buehvB2kevP6cZh84Sh8/E0tinyZ921nW59V2MZ1s31PUJrnQZHP7VxTtNe1en/3mzeX47gHZrU5+OTOpgiWbW/CuU98dkhkIz40Y90e12a94Y3lAMzfuM31rc4DzFwuOKYflt/9HQDAAxOPBgBcc8ogPP+p+XD+y21NGQP33jx+BACgosCLviU+lOZ7MLhXHqpbYvjt/67A3HWZZX/2hY6W30kv63Ti4DL84PgBbb5nV3MEL1jjEdjXHq1xFbe8+1Wb58j7/rsGN7+9okPrRXtu/75ao70qfbTv9O6sHZX+niJf5o1rR9nBo/SLEnOArNTD1+7CaI+y2Fn2iakztfp++LeF0HTDCVCVJdVntS9aju5fnHIzELaCdAU+F4JRBZIoIM+TGOSpPhjHlvpWlCZ1gQTMbkG98j2Q1dQT6EmDy9AYjqO6JYbB5floDsvI90q4cHQ/DKsocPaHHUC5aIzZJanAytwqzfcgpuh4cs5GLNrSmNKltMDnQl0ohnyPeYHWGlMxonchNta1ole+1+n+sTsQRZHfriGcPRsyKmsd7Lqe6dGP16O6JYrhd+bOePyzVWsxFzP4Zv5/KKaiNa4ivwPlCyKyOdCXnXGoWt93qTX4l80Z6EvRnWCdz8qUtY/jmKLDawUiKgrN/VfgdVndzs1umQNL87KuhyTaAybZwdhEsDPR5LK3V5ckIKpoTs1Ft2RnTlpd7F1SIlPWZw44JIlmpqus6bhwdD8EoorTld3rEnNm8AkCMOeWMzBuSC+rtqhdvgAoL/CiOSJjZ1ME/16xC9NX7cZjHycuwrJdGmg6rKzQrt1cpdddTielde0HEuULagIxLNve5JQLsYNi9gjvRT53RvmClfd8FxUFHme7zXIhGlyi4JRZEa2SBS5RTMmUdVt1g5PLRJifISYNtKZD1c0Araqb5Qt8bhGRuGoGZa3jLW5l1spW3dr0faLpVvBX1yEKgKbpzkOwP5w70jw/xRSrLq3hHPsel5SyzWZWuF2+QHAC2pqeyL59+fOtuPbVzmXPS6KA2pCMaSurM16zz612kLWhtf0Mlu88/imWdKGGmD1IV1zV0KfIh+akAFJJnvnbZtdxNgyzXUYVDSV5Huc3qiYYQ0WhF4PL86HpuvMbUWMFZcOy+d0V+90ZXeqSB1MMxRTkeyRcNe6wLpdwSA8sdrZdffB1NU5/aF7KtHDSAGVb2vn9tWttN0cU5HkSN/52QB8wv/t8K+tb0w08/0kiG3rFjmbMWF2DUf2K0N86V9r72Q7O/unDtfjVP5dnLLsloqQMEgUAJ/5pNnYHovjW0DLMuPk0AGa7HjekDF/eOR6FPjcGleXhlGG9UNUSxd8/34raYAwLN5vB1pveWmmtVwsA8wFbTTCG4x6YhZ1J2WHff26B8/92EMR2+sOJ/fnh19WY/N81Ka+v2x3Ec/M343+X7IDPJaXs49aYigetm9N1NSH8/fOtCMVU3Hn+KJx3dB+U5ntSRmXf35TkuTt8Yx+Oq3hv+S58dc93UWw9EMn3utCrwLxWsq/daoNx/HvFLoy46yOsqgo4XVyTa7Unsx98HHPvTKzdHcSMdrpot8fuGbWn5bT2Fy0RGcfc+zFu/dfX+Nv8zRj8xw+dc689GGF6hr+tOSxj7KBSzLz5dNS3xjH+sU8xc00iCPLZRrMdTb7wKGfahtpW+Nwivthinq+r2hjwL9nv3l6JBZsasaku5AQB2+vSu7/a0RSBRxLxvac+w/3/903WeT7f2IB1NaGMnlGAeU7OT0uISR5XIZn9O3bK4eW49ttDMHZQKb7aFciYzy2JeP4nY3HK4b0QkdWsI9gHYwpG9inEmIElKPa7UROM4ci+RU7W4v4q272JrhuYuaYGf/jX1+hrnbPHP/YpFltd5JMZhoHXFm7DJX9bhPW1IVz76pddeggcUzTn2P3Pyv17kMbn5m/GFS8syvl6fSjeZrJMXTCGC4/ti1K/yyyLZf3+t0RkBKzrLFXT8Z1RvTGoVx7e/9UpOGZAMcryPdg65XsYM6AEgHlfsbm+FcV+N3Y2RbHxT+dh6o/HAgCevep4Z3nHDizGuCG9MGZACSZfdBT+76tq/N9X1fjZq0tzZvnvTcPv+ijn9dPZj853yr+c+ch8Z/q3D++F35x1eMq8yWViDMPAjsaIM5j3+gfPRUWhmfhz9OSZWLylKSPBJqZozj1Jc0ROuU6jnsWgLDmSg1aAeTLs0kBfafUf0zNnO8PO1G2OpAdldbREFHy6IVFP1Q4c6l27V01kAHbwQtowDCzd3oxgNNHFKt/KVAPMwWiGlufj6H7FCMYSXV+awzLckogCT2K0ba9LdG4kRNG8qLcvlGzBqIJivxuz19bi06Q6soU+s2xDTTCKIeX5aAjLKPK58fSVxzk1XvWkAaIuOLYv7vieGXSpDcZQludBTDUDdiu2N6dkIOd5JNSH4s6gNXWhuLmM1jh6F3nR2CqjyOdCVXPUCSK7JSHrjf7/vLUi4+KyNa5mXMy/tWRHRgH2p+duQjiuthlgq26JtvnETxDMm/6YokHWdITjZpfh9oIbUcUc6Cuu6lZg24BhmDffyQFdw+qebwZhzUCVOdCXagW7BCtT1jymKwq9aImYZR+8bgkRxRw0rdDnwvzfn5mxHpKYyEgEzOxNAPj24eUp2RDZuEXRqSUbVcyMX1VPHejLJQlWfUazlIZLNDNlZVVHoc+FlojilC8o8rvbbCfDKgogWe+3r3V1a4T5iKxhwaYGTHrnK2ysbcWbS3Y6348tGFOdp9e6YdZSVrvYsO1Aqqrpzs15MntwqnSCAMxYvRuXP/8FDCM1KGsf30V+N3Q9ERxUdd05/ymaGdj12XVdBcFpZ/Z+lUQh5SbJZZW98EgiNCNRL87rErF8RzMUzUBc0ZzArWotw+syg/8+lxV0TwviSqKZhWifz3Wr3qudKeuyjgd7XX555jB4XaJVCsH6DEGwyhEkvlPA/Ax7f9gBbnteu3yBqhvORWFHSaKYUis5OYvAvmC0gx92YMCW7TxQkufGFS90vma4J6lkTFm+J6ULo/3A0a4tWxuKOeVHiv1uJys++Ya02O9GdSCKoRX5TvZk1CpfMKhXHlTdLHNiH1PJg0ZtqmvFz88YhotG98v6MOnoyTPb3R7798nOzuxIxu21ryxxbhSbwjIUa93s7apNGmTmuteWYuXOFkz5aG3GoFS6bg7aaQfRkh8AqlYZkTX3TcDAsjznPNQaUzHlo8TnRBUNG2pDGFDqdwJzyZmtU35wDDbWtaa0dfsG74KnP8ebi3dgV7M5KIY9OM7KHS0oTMoc61Psw9s/PxkVhV4UeF3oXeTFAxOPxqbaVjzwwTcY9+c5uOrFRFZgIKqgNhjDtF9/G25JcAYMe2LOBgBmreVivxtDy/MhCsDnfzgr5772uqWU76S8wIN7/+8bJ1B4VL8inP3oJ/jek5/h0w31+HDVbvxo3GF447qTcNkJqdkyf/vxWJw0pGy/ytpMP9faDyw7ktFo79fiPLfzoD7f63J+F2VVx5OzN1r1m1dBVnUEogrmra/DyD6F+FtScD+ZfZ4NxVSc9+RnTm3grrKzBbOVcXrji+179Nn7wpKt5rl71a6AkxG+fEczXKKAnU1mwHRVVQAj+xRmvPekP89B7yIvDq8scM7XgFmqJD+p/Rcn9TpIL41kZ1Irmp412/mOf6/CG19sx79XmEGsHU0R59zaE6Osp2e0/TrLA6A9Vd0ShazpqGqJOuXIALMnn91Wfvz3xfhkQz0G98rHYWV52PaX87Hlz9/Dd0b1RktEzuilWFnodZJXktlBGttVJx2GC47tm3W9JhzVB6ePqMCnG+sx9sHZGa8Hogr+eN5I3HD6UGhWdv+vzxrWbUHZnipJMfzOjzKuad9euhM/f2MZPvi6GsGogiP7muWKLn/hi4xBHf/xxXa8ZPXSLMlz44oTB7aZHZzrPPDKgm244KnPnf/fW+6atqr9mZLY272zKZrRnd42d10tLmmjjMvirU04fUQ5Pvr5aMxeW4eZVg+RO6etxgSrFFIwpqLE78Ynt56F4w8rdd4rCAL6JNWf3tkUxZ++f7Tzml0K6fyk43hknyKcPMxMFjl5aK+Unj1XvbR4n2fVGwawMcdAelvqwwil7ee4qqHQ63ZiLC5JwIbaEL7z+KcwrAHSa4NxnP7wPChqIgFH1Qynl1ZVSzSjLMcd76/Cbf8ye2wFY+oeJdZR5+yVoOyzzz6LwYMHw+fzYdy4cViyZEmb87/77rsYOXIkfD4fjjnmGEyfnlrryTAM3HPPPejbty/8fj/Gjx+PjRs3pszT1NSEH/3oRygqKkJJSQmuu+46tLZ2LYPy0JLUPdKqadhZyUGGbX85f4/Wxi0JKYPRON2HFXMU85++kjiW8qyLjWiWLoYdYd94dzRT1r5QDEQVJ1NoeO/CRFBW1fCf33wbg3rlIRRTsbXRDD42RWR4JNHKSDQ/QxAEfLWzBb97+ysnuFziT8uUjSoosi5ck4OeBT4XWmOqmSnbK98JotkMmJlY9sVYoc+NG08fBq9LQktUQUmeB3FFx+iBxZi9tjblqZjfbQVlrYu7TXWtGN7b7BpfWeRDYziOikIvdjVHna68drAnnaLpTmaY7Z9fbMeFz5gXINUtUbyyYCteXbgN/8mSHZcemE/XFJbb/O7tLD77Zi4cV1Hgbb+mbCwpKJvMHjANMLucAuYNpl2+wMyUFZ0R7M3PMmty3nTOcOR5JASi5qBAduzJHgjMLlmRzCXaQUFzZlXT8fvvjsCFo/s53XNycVuZsvb6uKygsdvJMBPhsgK3BVYwRBLNbvjm8eRGS1R2nmYXWsHV9rjE1IG+RMGugWtm6pbmu52aaACsOrdmduLPXl3qvM9rddXPJleXVJt97lhdHcQFT3+esjzADkymvscO1Bb63IlBsaREXVY7UzTPY2aN2rVk7SCpvdw8t+QcC3amrM8tQhStbYUZfLT53OYFkz2Yln1z73GJmL6qBlXWjVpioC8z2Ot1iYipZgA1Ub5As2ogm4FiAwZ81vdtZ8o6tWutLNjk873XJSEUU53zgWidh91SoraxWxIhCkLKTaquG8726Ya53+yM8c5IHwDNziAdWOZ3ApI1wRj6FvtS6tU1hWUcnpZRH4wpOPXwcpQXeBCRzazX9CyU5NqTg//4IRpb4/h4TQ1aIorzPZRZAx/aDzN6F/kw+cJRzmesrgri2IHF5v4SAJ+VuXn3BYl5yvK9qGqOYmh5ot532Mpy/uN5R2JQWT5G3j3DOaaCMdXZv9sbIxhWUZC1N4Ki6U5GbzbpPUHsmo4d6b0wb309TnhwNhZtbnRKLQDA6Ps+BoCUjKmtDWFMfHYB3vlyZ8pDpP9+VY3vP7cAoZiCigIzCJD8APDwygJcdsJAJ7sr32s+0ArFU9t3VNZRE4ih2O926qva+xkAehd5EYopaAzLTrfnXS2pQftT/zoPN7+1Et9UBzF6YAlW7GzJ6JliK/C5kOcxezM0hBPb6RIFnDGiAt8Z1Ruj7/sYn29swIBSPwRBcGq3ra4K4skrxuDskZUY1a8I7/zi5JTgL5Aol6Q7D3z0lBraR/c3j6dgTEHfYh9+9K1BAIBvdgfx6sJt1sMfAacNr4BumEHbrVO+57y/osCLprCMD9Y04P3l2TOvqnsgcJXstYXbsMDKRBp2x/SUElH28WqXnfrDv77OOCYNw8Dt73+NTXWt+NbQMmua+Vqh14VTh5dj3JAyxFUdj8/egHOtgdo++O2pAIDN9WGcPKyX08MhWVTWnGBhYZYu213R0BrHS1efkNHrRdF03D1tdUoG9f7g5rdWpAQk0s+NS7Y24Y7vjcT62pBzbffDqYswsCzPCXra14LZXDS6PyRRwKxvEhmyzWEZwyoLMKJ3QUpwduld43F0/2L43BKevGIMAKDaasevLNiKcx79BK8sMI+VXc0RGIaBNxfvwN3TVuPUw8tx64Qj8NJnW/H95xZiVN8ibKgNdbjWdTK7pFe6ZdubcEJS/VVdN5x6rnsqFFMwY3UNYoqGcFzF/RcfhRMHl6JPUWKcgVH3zMRnGxuc9lRe4IEkCvj0NvNBjygKKMvzoDmiZFyHX3fqELz781MyluuWRHx1z3edv085vBzXnzY0Yz7biYPL8HVSJq2i6TAMA4GIgnv+syYlwA6Y9zLdFZQ9/oFZ3T5omH3spz+0t3+fRcFMWLB7xADAi59uwbLtTTjynhkAEgOc/e8N38JKK5M/GMserAxEFdw9LXsdbUlMXNPWh+JdqsevaHqn3mcYhlP3uaOS64Dnqjn6h/dWodDnSrk+fG7+Jmypb8XuQBTVLdGUXiN2Bv2HX+9GS9S83ntoxrqcgcpe+R6cM7IS5db1xJF9i3DlSWYd5BG9C7HpT+flXH/7gbm9/H7FvpzL6YrOHqO6db+Y7bfB/i7v+re5z48dYF4ThGKq00MVMEuGfPdx81y3cHMjjp48E/9cbAb/gzEFo6yHCsGYgjMfng+3dZ0xY3VNysN/r1vEJitjNxBVsj7IoZ7R40HZt99+G5MmTcLkyZOxfPlyjB49GhMmTEBdXfZ6KwsXLsSVV16J6667DitWrMDEiRMxceJErF6dOAE89NBDeOqppzB16lQsXrwY+fn5mDBhAmKxRMDnRz/6EdasWYNZs2bhgw8+wKeffoobb7yxpzf3gCYgM1O2a0HZ7lsnl5To0uC3RowHEtms9nnPzhZ7+srjul5rT+lcpuxu60l8wMqU/d8bvoVTki7849agOIU+F0IxBYqq46h+RWamrFVTNnngEDsL1L5fS8+UbQzLzo3jwj+e7Uwv8LrQGlexOxDFIKveXEHSzcWQ8nys2NGS0ZUJMG9IzBOumcU4rLIg5UbZ55ag6obzeZvqWjG80syG6F9ijkReWehDVUsUJfkeCAJSskGTKVpmjWJBSGS7fVMdxH3/9w2K/O6sNbSawrmfOqua7mRP5yJa2Xv28lrjqpWBmfrj+dnG+pQfxqhsdkVODXaY9WLjqo53lu7EhCc+RWvcHPXeLl+g6gZ8ruTyBWaWrs8t4nffGeEMduP3SNB1M7tQ1vScA7NISQFOwDxeLh7TP+f2JnNJZsDV57K6t4uiNXCZebDZNWVjijXARlQxM2Wt8gVFfvNY9VrtsdBn1p01jLZrhSYP9GXXGhWERK1olyimHPeaZu4HKSlooVm1aLPVW1I0Hcfc+3HO5ccUzQmg2he6Y+6flTKPed5L3QYz4zNxMaQbSKmPq+k6ThpcZh5TVtAWsLL9BDMrWtUNKwtadPZFXNXh80jOsWjug8S25nslyJpZmkXTEwHe5PNwTNGh64nsYVEw22ncyoCOKYkMWbdVx9ZehscKXum6kVJT1gzOppY58LnFlAsyO1BtZ1kDZjDfPkfodsDWSMyr6ea6dzUBITkou7muFXedfyQenHiMM70uFMN3RvXG8qRMwNaYmnFDsnBTA741tBcuGTsAu5qjeG3hNqfbOQA8O28Thtw+HbpuOLX8vtzWjC+3NeG1n53knI9K8tzY1hh2zs1+j4Rrvz0kZVkjehfiwtH90LfYD5ckoDTPjetOTcxTkufG7kAUwyryUWsN9BVVNOR5XZBEwcketQOaD3zwDX5pZWI1R2SU5rvNB2pJ5zpNN5zSLvbFdTCmpPRCsL8z+3W7Nm5c0TucIXLli19A1YyM3jBqlgcmFYXelO9va30YVS0xhGKqcxOV/Ftz5UmHORnHAJy6yOlZOBFFxdLtzfC6JedhYXKmbO8iH3Y1RyGrOn70opn90hpTcenYASm/qfWhOJrCMk4e2suptZZNgdeFfK8En1tKqeV2xogKeFwihleaDymXbm9GL6um7YWj++HVa08EYN4oCYKAloiCsjyPE2AxDPO8M+oeM7t5TXUQrXEVCzY1OL8B9sOcH407DNUtUTz/k7EY0bsQpw03R4/+eldLyrraDxCTg75FfhcCUQWfbQng3zm6w57yl7k9miX08Mz1KW30lQXboGg6dgei+PN0MwvaDmxtrm9FJJ4aPN0diOF/l+zEKwu34rdnm/VjTxpsBmfzvS6nzf3u7ZU4e2Ql7r/YzJgq9rux7oFzAQA/sYLZQOoN85H3zEBtMAafW8R3RvV2psuqjj++93XObfr1m8tTSmUt2dqEUEzBy59vxdaGMMYcVoKWtEBUo1VmJf3h9N4y6Z2VKd/zP77Yjk831GPaymrnXASYJTnsa6AdjRHMWVeH6081g3Q+d+qDlP9dvANPzdmIzzY2ZK1ffM7ISidInkzTDVQUeNG/xI8195/rTC8v8JrnQ4+EoeVm27J7xEmiiLpQHPf93zcIxRSc+td52N4YwblH9cGph5fD4xIxekAJFm5uREmeGw9MPAqfbqjHupoQ3l/e/qBZyZojMp6cszFjekOrnBJsC+UIvHXFql0B/OIfy/DAB2aPsqtPHozJFx6FYEzBf7+qdr67qpYotjSEcfqICqeOdjJRBFbubMmYLghCSu+EZMV5Hc+GK/S5nLEqdN3AlS98gX8u3oG3vjQDe4eVmfchAszs3GK/u1vLFzRHujdb1i4plz4YmR2UVXTzQfjpw8ux6t7v4ldnDsOjszY4mfuGYTjbbgfJCq3yT+kaWuN4YrYZ7P9yW1PGtUpLREFY1rBqV8C5t+ysH/5tId5c3H5Gvq4bqG6JOr/T6QHBbHWkAfN3xh6E7iffGpT1ntk+x15y/ADMserr/mdlFR6asR5fbmvCyVPm4sNVu50A/sybrAdofzYfKLqtB99vfbkz67EMmNfUf//pifj24eU4Z2QlBvfKx5QfHOO87upAMKJfifl7fP1pQ/HOlzsRiil7XDZi0tsrMfSO6Zi7rhYz19Q4vcQMI7Mev+3Wf32NmKLjwQ/X4h9fbEdDa9zJxh52h5mY+M1us5eC/VB7e2M45VosOTN+sdW74em55sBpUz5ah4d+eCwAwO9xIapoznXYn6avxYqdzXht4Tb8dcY6FFm9bwHzGq29xCXqPj0elH3sscdwww034Nprr8WoUaMwdepU5OXl4eWXX846/5NPPolzzz0Xt956K4488kg88MADOP744/HMM88AMA/qJ554AnfddRcuvvhiHHvssXj99ddRXV2NadOmAQDWrl2LGTNm4KWXXsK4ceNw6qmn4umnn8Zbb72F6urMDDwy2eULEiNq610a9Tx90KE94bFu8gHzAtvOlgvGFIw/0uyynd413w5cNbbG8ebijj/9i1sjoYdlNeuAHumaIzIGlvnRYmXK5nslCIKQsv/ckoB8rwuRuIaYqqNXgRd1oTi8kpgyCrdNEMwgxg+O65/1YuvIvoUYUOpH3+LEk3OfVSe0Lhh3BgFJzrK1M1nTR20GzJud5HqAQ8rznUxdIFGrr8jnhiiYXSMGl+fh/35zKk4aUobmiIKKQi+qWqIotS7qYoqekrkEmN1YFmxqzHhyH4qp6F9iZr7ZN9YuKyBms3/c02stJotY31dbF2v2gE5BKzs1HFdTfrTtrkbXvvJlSuZDRLEG+kr7YfK6RMQVDbf9y7xxqwvFrYxU83tXNTOYGHWCslZQzto3HpcZ9PJZwYVh1s29N0tAG0gECgFzOzrTPu26oHbGrNslQtbMruV/+v7RTgA2ruqoKPCivlV2yg+omuEE9M1MWTPbOhxX8dNXvsQ/0i7+kq/hkgOZugFIgoAdTRFUt8SsbUpdT80wIFkDSjnTNMMpAZCuvdITI++e4XTTTn4AsrMp4gymI2TZhXagNiKrTnZwcvkCTQeu/fZgJ6hpr5udDQyYDwoKvC6nziVgBsR8LsmpuTpnXR1WJWVn5HvMzG2PS3KWCSTa4Xu/PBmhmJKUKas7mbJx1SybEVPM+rXJtWRFKzPZm5QpqxuGU1PWPIfozoWwuUwzk9vOlJVEOGUT7ICyzy3B7zYHE1OTXrPLGNiBWiNrteC2CQJS6mc3RWSUF3hTskSjsoZBvfLRlBSgjFkPT5KD+F/vCmDsoFIcVpaHnU0Rp2zMx2tqoOmGM7hETTCG7Y1h/OC4/tjeaNaSHVqe79xQFnhd2NUczcgESmaXjhnVrwgRWc2ox1Xsd6Oqxawxa9/MhOMa8qxgx8M/PBbfGlrmPMRqCstOJmNzxHww53GJeOnzrU4maDgl89Dc7sdnbcDP30h0w3b2mXWutJf9s9e+7NTgmMEsN2qCYA4CU+RzOTcGfo8r5YbS7xERiMoIxVT0sm4Gsv0GJj5TwKqqACY+uyBleswKKo/sU+g8LExuY/2KE5k3YVnF0DumIyyrGHNYScoAiGFZRXVLFGMGlgDIPcDniN6FGFCaB79bSgnG3HfxUXjmquMwoneiy7YdDH36yuNw5hHm9YnHJTrtUxQFnDTEDCYW+d2Yvz5RhigQVXDj60vx5bZm5/haXRXAqL5FuOG0oVCSzsPP/2QsbvnOCPQu8uG1n52Ucx8C5j5+dNZGfLK5JWvQbE21ef7pag+jjhhWkY+dVomRXvkePDxzPb7e1eIEKS8dOwBDrN4hLVElY+Ate77VVUEnqPTOL07G6IElTu8hj8sM2M1dV+dkvBZ4XfC5JfjcInrlJ7I4j73vY6d3CwD8bf5m/Pn7x+DC0f3Qr9iH/iV+PDd/E976cqcTWNnWEMarCxLt5MOvd6cE6S97fhGembcJ93/wDT5aXYNe+Z6MB5YNrXH0L/FnDNSyt7y/vMqpH2gYBu6athqLrPqYrWkZ6ac9NA+7A1Gc/vA89CvxQRQFLL/7O1BU3Xn4UFnoRSiu4rFZG/D94/pj9to6p950Xcg8l6af+d+8fhzWP3guNN1ArwKP0wU5WbHfjZaIgv6lflx/6hDEFDPzLzmwN89qO2c+Mt8ZvCcqazhhsNnF+ZbvjMDoASVYsq0JZ4+s7PRYE7l6dtjnibCsYUt9a5cDhJvqQikD/AGJHmahmOqcS/K9Ljw7bzP+539XoDWuotL63dhc14ofjh2Q9T7hd98ZkZJd290KvC6n9mUoriIsa/jvympM+WgdLh07wDm/jz+yN37/3SOsB/3dF7xur9dcZ7VEZAwo8SMQVVATiGF3IIqf/H0xZq6pxX0XHYWYomP++noIgoBCn9t5UPiH98wu/8fe97FTxsP+/elT7HeCnNsawnhi9gaEYgpOeHC2U5bg0qmLMOyO6c7nxRQNz803S6zc939r0BpXO93DKKZo2NYYSenqbmbCJq7TtzaEMfiPH+KLLY045S9znfvde9NqmR9z78dZa8JuqU887B1Y5s9avmDicwtw0eh+OHFwGX7+xjK88+VO3PG+ub/se7mvdwWc8/dwK7PVaQNxFZ9sqMfwygJ8dNNp7W733396YtYemm2ZcfNpOHtkb/z8jKG4ZOwAzF5bi2Pu/Rg3vbUSs7+pxR/+lfuhXFvet8qoqJqBP09f69xrbG0IZy35AQDbGsM4/rASAMBd01bjhAdnY+onm3HZVLNm72jrOmXSd0ag2O9GbTCGS/62KCVLOfkhwFNJD5ROPbzc6vln/i6+/rOTzOSg8SNw27lHAAD+s6IaS7Y14W/zN+P5T7egLN8DwzCQ73F1+0CYlFuPBmVlWcayZcswfvz4xAJFEePHj8eiRdmLQy9atChlfgCYMGGCM//WrVtRU1OTMk9xcTHGjRvnzLNo0SKUlJTghBNOcOYZP348RFHE4sWZo4MCQDweRzAYTPkHALquHxL/7Jss++ZZ0zQoqg6X0Pl9YGffdct6JV3SFflciCsqdF2HrOp4/sfH45yRlYjICnxu0dkOwzCXvaY6gDv+varDywpGFfQuMruWjrx7RrvzR2UVvQt9aAnHEY4r8LvMdbC3HTB/DPPcIlrjCqKyChgGdjRF4JYE5HnElP0EmEGZIp8Lj1x6LDySkLK8n58+FEf0LsCnt55pZSia0z2SgKismjVMrZuuQq/kvO6VBNSHYsjzSBnbAJhZf3Ywfkh5vrMvk4+LQq9kdo2WVbhFAUf1K4TPJaAlIqO8wIMdTRFUFHjgEgUEYzI21rXipc82O59jX8CZtRITy68LxnBk30I0tcad9RHSjrmwdbPQaP346LqOuWtrU+ZptS7Wm8Nyzu9LFMygUjAqo6LQg9a4ClEwAJj78oQHZ0PXdeR7XdCSv+e4ikKfWT82+aGFHQCzNYdlJyNREgTEVRUel4CIrMIlmsdCVFbhdZnfq0s095/PJeKqkwbitWvN85XXJeZcf1Uzv++YopoBuaTjxw4IZnuvHbOQRDMoK8GAag3kd+WJA53BwmKKhl75HtSHYpAEAYJgZoH6nMxWwfyh9kpojalYVxNEQyjuDBqmqFrK9ycIgKKa37mqaYBgDuKRPsKqbh2DqrVNdn1nXU8MaiWrasZ2RaxjI9d3DgC1wWhKmwSA2d/U4H/eWuEsN/0zzLZr3oD53RIU1Sz5YL+uaBpEAdZ3okFWzIsWRdOtA8SArGoYN7QMxx9W7Hy+nSltHotm+YlkXpfoDMRmZ8nZ0wHguIElCEYVxBUNbivwKgDwuMwsZzPbWXMebNgxe8l6IGEHwTVNh2JlxqqqmZ0dkc32kDiv2CUYzDcJgJUpa2bVGoYZ7Pe5RSiqDkU110m3jkNVM6DqunPst/U9ZT33G0bKcdAQiqPE74JbNANI9jm4NM+FcFzJOF8cfudHiFrTG1rjKMtzY0CJD9sawli3O4gzRlTgxjeWoak1hu8d3Qd3fG8kvtjcgFv/9TVG9ilEbTCGYFRBWZ4btYEYCn0uGIaBloiCQp8r5/HmdyeOk4GlfhxWlpcyT5HPheqWiDW4o4YLn/4cwagMv3XeLfK5rOC8WV86IqtOm1pTHUSJ3wU72ak1am5fMCkwEJNVrK0OICprKb8TUevC+rUF26zuyub8dcEY5q6ra/O7SGaX/0j+rdMNAy9dPRaf3nqm85CjwCshEk+c7+0BvgJRGYVW9nW+J/u5zv5XG4wjpujoW+xLtHnrJvXwinycPrwXAKT8Vhb5EgHaYFSFYZjZfnluKeVxcVTWsLMpgj5FXmx88Fzce+GorOtwzsgKjOpbCPtZ2es/O9FadwluUcCEoyrxn1+fkvX4vvbbg3HS4FKzXcvmvrjvInM5N5w2BPPXJ3qMtUTiTuC8LN9j/T5G0bvIixIr+yrPOkZ8LhHFfhfWVAdxdL9CZ3mTLzgSj116bMo6FCRlxomC4JyP7X/2oDWnTJlrtp+YkjFPV/+9tnArnv9kE3oVeNBg/cYPKPU755Nbrbp1d35vJCJx8/zeHJbRGlNSPscucQQAvqTfx3//8mT0LfKa52TrYcwPx/Z3znt51vFVXuBFvkfEl9ua8cFXVVatvahzDK+qCqDQ58IZw8vx+R/OQlVLFE/MNm9qq5oi0HUdn2yow73/9w1W72px3heMJq43AOD5T7bg24f3Qi/rhtawri00TcObi7djc10IR/QpwD8Xb8c1Ly+Brut44ZPN2FQX6pb93dY/TbNKvrREcOnUhbjwaXOfbqwNYUTvAgStc4ldS/CEQaVOUOm4gSXQdR1+l4Cl25txeGUBinxmSY9nrhwDADhxUClEwSyzJSsqTvrTHJzx8PyUcwsAFPrMdtO/xI+TBpfhd+OHZ6zrJcf3xyXH90eJ34U7vjcSA0v9WFsdcK4jDyvzY8WOZpx/jJmB+011EJJg98YSsPyuc/CjcYdBFMzz1eGV+fj3iirM/qamw/srbAU4NtUG0dgac6bXh+LI80j48JsGnPfU52hsNR8wy0rm9Um2f7e++xU+31iP615bitcXbUt5ze7N89WuFpwwyNzneUkP6d9YtA19in1ojSnY2RTBUX0LUZbnyVhGeb4HC/94Vo8dS36XiF3NURR4XdjZGMYRvQucB4TJv41H9SvED8f2h9+6B+rq8jbUBHHylDmoscrQpJ8fdF2HrKj4aFV1lz6/ORxH3xIfHvxwLb41ZQ5OnjIXn21sgKxq+Mm3DnP2vz3/tacMsh64izhhUCmK/W6suHs8Zk86HUU+8z7rzBHluPntlYjJKjbUBvHE7I2oao44gbfHLj3W+dzm1jg0TcM6KxMSSNTPzratuf5d8PRnGHn3DBT5XHht4TbEZBUffl2NG15firumrU46hs1jtsb6zlpjCr5/XD/4PRKmf12Nr3c2O+21NhhLWYamadja0Io7vjcSL109FnluCbO/qc1Yl693BaDrBr5zZAWO7leElxdsdR4Iba5rxVlHVAAACjyic51i37/MveV0PHn5aHy9qwUVhV4c0bugR47jEZUF+PnpQ/CHCUeg0CulXI/PXFODBZsbnHkbQrEOf67twQ+/wfbGCGTVnB6IJq6donEFf/9si/OextY4Hr9sNH56SqJHxxOzNzqD0w60fjd/c9YwnHNkJeauNUs9JJ8/j+lfjD9//2hcn9QzCwCets7R+UkxgC/+eDYuOb4frjppoPk9+CSUJWXLiwCqWyLoVeBxfhv29F9jawxqN11bHGj/OqpHh1RraGiApmno3bt3yvTevXtj3bp1Wd9TU1OTdf6amhrndXtaW/NUVqYOfONyuVBWVubMk27KlCm47777MqbX19enlEU4GOm6jkAgAEVREI1GAU3Hzt21CLSGEWhuglft3BPmeNh8iparREVn7KhtQoFHQquswSsZqK6tQyzPjVg8jvr6esTlOLZX1ULQZGd5cTmOuro6bNrV2Kn12LG7Ff0K3Vi0oaZD76trbEaxB9hV14z6lgiirS2oQ8RZfjRm/jfWGkN9cwhlbg1nDCnApxsbEGsNoKFVSVnOCQMLUR2Iw2UoWZd97fGlCDY3Ipg2PdraioaWVsTjMhobzOyBYHMDQtbdsRyLYGd9BANLvBmfO3ZAIaDGEY7G4HWJ+HZ/D7wuPWM+ORyArsjIk4yU13QDyBfN7sIFRhSSoWJ3k9lenpu3CRcMzzczDq1jIhKLpby/pTWCYo+E0x6ajz99z+wep8oKooLmzNcYNvfTrgYzo6emthY/e205Pv7FaBRZT/52NMVQmufC9t0NOLxQw7raMOZsbMavT00MfKIqCqKqgqq6JhS4BTQGWhFpdSEej6O21vyBq95di0KviLqmoLP8plAYejSEllAYiqxBh3lDGW0NoqklihK/yxzILaIg2hqyliWjvikASTAz0uLRMOJyHPVNLeiV70ZdnYTWQAxNrTEosTCaGhMjf0ZCQdTVZT7xjUcjUFQNhiajancdWsMRtDQ3woia+yAai0PwiFmPnahVTzvSGoKs6mhqbEA0LkOJJ76PYCBsBpnlMBqDEZT7gEhYh6brzvq1BlsgCQIMVUZdcwBuAWgKhBCX4xAA7NpdA2iJ4zcaDqOxCdgixtDYFEU0nDqoWyBgHs11dXWQZRm6KsAnapCtG+y6ujpEolFIgoCaugZ4lNTsj91Wl8u22mpVrXlRU9eYeOIfDrdC081jOS5nfoY9rbElBI8I1Dc0AJqGYKgVdXV1aGkJwq26EYtGUN/QiJh1HKq6joaGemiKjJr6JvTxGyhzq87nR2UVSiyCeCwORVHRPy8t+1rUEWgNQ9A1xDXV6ZpkaInPKPIAVXWN0JQ4QjEN+R4RSszM9A+HAghGYpBj5s1LKGi2mdZQCLKiIGoPaBOOIKJo0FQVgVArXIKO5mArgi3NqJOsLntWQFSNmd9ZoKUZsqpClWOIRFXENR2aIsAtGAiEWlFTWwdDUxGORqEqCgxVgWRIcBsuxKwIYGd+E+Jx2ckMr6urw676Zgwu0BGWBTQHQqirq0NdUwsKvRLicRlL1+/AYaU+1NQlMuBWbt6F1piG1TuboIRb0Mej4ZoP1+LUocW47azDsK0+iEc/Wo2vdwXw+9P74PYPtuC4/gU4e7APD82tQ0TREGxuwIbaEMb2zzOPF0VDLBLOui0zfz4aHqUVddYN1ZmHeXH6wEH/z955x0dRrW/8mbq9pG0aIRB6BwEh0qVJUREVO2Jv2L0qigJ2sV69du+1XEX9Wa5dbGClIzaa9J4C6dm+M78/puxOdjcFE0p4v59PlJ05M3OmnCnPec/zGsrKgVpsL63CwBwzVmw7gIgMdEwV0MnNoKREue4FhPHXzn2wiwwO1AJSOIySkhJ8t7EUoZoKVKuRvtv2lsABL3YeiHqC7i0pxTmvr0OKhYfbwuvbLqkOwi5yWLX9AFgG6JdrB2DDL7uqsWTLARQVF8fZEiRi9fYDAGTsLVLumeu27YGNU54ZvlDUn9POy6is9aKkpASDn1yNSwdnw8Qz2FNyQLcnKY+579VHQaoJxcXFWF/sRUV19P2ii9LfAb+3JuH5CKpC1NxP1uHRUzogGAqisJ0TqVYBn607gK0llZB8VTiwv/GRi50caoKXqjIEqpX9yBSAzy7rHVeHKwamoaSkBFIogBp/0DCfDYTx2tJo9NL9n63Dvirlo62y1of12/ZgzdYDaJdqhq+qTN1mOUr8nLrPynHwV5WjpMZ43kr80beEWB9VExPGll37dJE3Ism459P1AJQI1ZKSEpz737UYlO/E5YU5Sa10Gsv7q3bAF5SQ6RQhy8q14lAD+y5+dSV6ZdswtnMK/NXl2LG/Gt/8uhUVviDOemEpXpjWBdPfXI/F1/RFG5cJuyuVc+SvrkAJG++B6+ZCuHBgFq4akqUfZ+366plpwf79paj0hTDzrV8BANP/sxIvnxX1iGQCNdBOz4A8B1btqkbvHBuWbdyNFDYN5RXKMf339xvRM1uJuN5dUoaSNEYdsaC8Dw3IseCxye2Ue0UgiCn/+gGn9crAvC+3AwAuHJiF9347AFlW3mUe+GIDft9Ritnj2v2tY10flf4w7v9K2f6YJ34EAHRKt2D+yR3wzx92oUOaBbuK9iOF9eGDX/ZgTOcUBMIS3l2uJkYLKe8JsqzYu2gWL53cLAZkchiQ54CTDeCHa4/DbZ9swR9bjcN/Y6/7YG0VSkoC+PdZnZTOQ28lSurYKA7O5g3L2bkwLnt9pd4+emdZsWprKR49tSN27q/GCW1t2FXhB4/oe6NmDylyDBAKYG+FH5e+vhrPntEZx7WJRreHI4oHOssweO7nPZjcIw15bjN2F1djcL4TC5Zsxn+W78N7M3qijduETXsPwGMTIAX9CEVkPPS5El24e1+xwY4lEZIs493Vu/Wh30w4+v4lyzL2lZbjyhNy8PySvcixKvdUrRPq0VM64JaP/8LoTinYd6AK/kAAYqgaJ3d1Nsu31sHQPtWE1Zv3wsZFcKAmALvIoW+mEFcff0jCvrJq7NlXhDJvGJmO+OjeZIQiEsY9vQYA8PVv2wEAe4r3I0s03rOLq4O46s0/sOyG/k3ejxcXb4crwaCNtuo30+B8Jx49paO+XxYAwwuc4FgGn68/gIgEBKrLYQdQGhNFOqVnOv7auRc+Vbg/6Z8/4dphueiVaUYHZzTgaMvuIpz5/FYUVQdx4cAsfLx2P3KcIip8IewpKoVNMr43J6LCF8afe5R7lMvE4s8iHy59dRl+2hodibV55144zTx2FynTVmwugoljsLuoFFY2gi1lPly9YA3O658JzzDl26miohIlJdGOgUWbynHHZ1vxwKQC9Ey1Ic/CY/ZH29DJzWJER7dejmcZTOziQNmB/XCZGKRaBWxQ5Zf3f9mDSwZlYzGA2soyVFdVKYEh6kgtK4DOLhnXv6PYPByq63tAJoc549th3pfbsbesGowc/RYe/ORqw7WlBRowdd6ZtOlOE4edavT0a0t3oF+mgJKYb/4p//4DRdVBTOqkjG7dXe4DF6jGxE52vLoEyE8xYUd5ANlOEfuqgkgzyZh7kvJcMUW8WL21DAVpZqRxPpSUKNfX2PZmfT9e/gmYNSZfCbZR3x/8VeUoqY3W11+t1Pe0Xun4Y+cB5Kcqy/fKtmHt3iqc/cJSTOiWhk9/39/o98P6GPLUajwwsYPhOjkWqK6ubriQSouKskcTs2bNwk033aT/rqqqQl5eHjIyMuB0Og9jzVoeSZLAMAxEoQwmsxmiCXC4UyGYSpGVmaH7jjQWszOE60eH44Txg6F3OxnXOR144PMNSHNasbZMxrJVRTCZRHg8HpjEXbA63UhzVerbM4m74PF4wG31gWcZpKdn6NkJ64MvAzplu7F2r+rbkpERd8ONxbQnhLYeCZIgQubCaJuTCadZgCjuhNmZApvFDI/HA9nsh7y+CqLVhkyrCGA78rI9GOy2oLBbHjzqMKP/u8qDgfd/i6xUZ5OOXXbIhDXFIbhsEX252E4LT2oYvxcHkJnmjlvv65emgWMZXPCfFUhxWNC3U17CbRS0yYLLUQ4Tx8ato2teBmziPnRtl4O0Pyqxu6oC3940HPd8uh5L94axcG0R+rdNwc3jOmPNznLD8iy/BwXZToR/L4VgVYYvms0iBIGDbHbivV/24BQ1e6ZPUqJ1nSlKdJTFkQKP2ns4+8vVKPeGIQkWeDweLNqxC2+sLsa8qcdFz5dpJ1iGgdlqR4bLhjADpLrdMJlqkZqu9No6UlKR5bJC4sRoPbm9KGiThcjqA7BZRT0pT2Z6GnbWlOG4/BRMG5CHK9/4BZ70FADbYbeaYbLYkGoT4QvtRKrbBVtJEKxoRWa6Ax5POkKiDzXBv+Cpc17a52TA43HHnQOHvRIyUw6nzQqHOxWcUIzsTI8+XJgTtsNqERNeOxkHlBeFVLeiYmRnZULCBrgcNr18abgSYUlGm8x0BKRiOO02uJwWSDL0MlmedJgEDg6bFeBNsJoEcKIZJlHxP3W40+Cyl+rlXc5a2BwOjHluJd6+bBCcVYp31Ph//gSWASw2O1hGWb/ZtF2JzLRYEJYUAdXj8UAQ98JhFuB0p8DjMWZ3rmFq9HJJEZVrxGyLDl12O536fpnEXXHr0KbxJgssJh4pqWmwmPfAbFWOl9XuQ1qqFY4qwJWSCpFnYRGUaGqPxwObdS/MNgfcsohUdXimx+NBICIjI9WFynANGLYWqSnR8/zNjcPw3PdbEQxLsFsVb1Rl6HsN7FYzgCp4PB6kOovBmW1wO2R4Iz7YrCLSUpwIRXYhIy0VEZQg1eUEsBfpacpQ6RS3C4JQDp6PAAhBNJkRRBgWE2C2WGGzeAFeREZGOjzqMGLt5TLXkwZgC9LTUsEwO+Cy21Ed8gIRGalOOxzWIESzBSlpabBZ98BkMsEUZsCAgSBycDks+rCqptzXTKao9UxGRgaC2I8ObbKUjoMdPuXa2OJDZpoVJlM1pr22FhvvHQ9TefSeLVidWLR+D9YVe5GTpdwTJ/cqwYHaAHp2aIMRXaqwakcZTjsuD+3aZMNm3QMHw6B9myyEmT0wiTwyMzNRHYggK80Fj8eDF6f3h8dhirsWAaAxe9cuIKK0djeyM1LRwVOKv4prEJB5tMlM19eZ6iwBTHakOyzYUR6ASVTadWFBGtrkZOnDeLfXMMgJm3GualOQZhPhcKbAbuJR7gujXbpdP+an/HsRaoIRNfFjBD1yeXTPceLPIh+AEKyuVKzdUwWXRUD3HOM7j8gxCKoWHdvLlORq7lTlPhwR7SjISoHH4zHYjnjcduyt8Ovb500WWE0CIFiRm2rF1ge6ojG8e8VgPP/9Vsz7Zg/W7q1CFzXDe+y1lJWeEndtfXbtEJz+/FJE1CiFXE8aOmX5MSA/BdX+sCLKHgigW7scmBopPp7Zv43+bM2qExhQ37l32PZDgt9QxzRJBvCb/lsTnADgl901mPSSMnRy/bxxev3a5mTp7zIn9raBES3IzjLWoy5aO15yXT+88ks5OIsTHtUqZ1Ox8rEwbUAb/N+q3fB4PCj3/Y63filBr3wPzh5ozMD+z2834frRnerdXizpzp3wBiIwiTy27a/F5Jd+R//8FNwxsSse+HwDSr0SPr9OiTJeX7wal/+fMoKi3BfGC8tL4Q9LEOwp6JLtwu5K5eM42fma4PFgQowes+KOE/V312enK8f9p1tHYuj87/Qyl76jBId0z3ZgVO/2+vveG5elo+vdX8JiMqEqIsDj8cDuUISRPdURMHwI43tkwmxT2teuMi86euz4q7gG158U9TR02vbg521FyHJHnz3HFWTitZWKOnHCPxWv6E/XHcCT5w5s1HvqwbBp8378sLVSt9sBgJ55Keien4U9lVtwQWF7lIc5hAUHwpIMq8UMORBBmR9YdPNwZDrMcV6kq2ePhlv1L/6/q2KenaZdKA0potuim4djU3GN4brvWZDb5FwVHXIC2LdsH/JTrfjqxmF48cdtWLJ9GzrmZeOja3MAKJHvNYFwXLKxT64dCodZwIEAg/d/2YPdXhYnxdTn3z9tQ4bDhFP65GDpjo3YVyvj+fOPw44NNTi3sADXqd7jZ7z6Jx6c2hOlPqC9xwFONEPgGKzcWY0pfXPgcKfqw/br8ux3WzCsUzryU6OJ3yb2zIJoturH5qwXl8FtEXBq3xxgyV7065gLj9uit9+pgzvjrV8P4LiCDPyyowImk4jc7CzkZifc5CGhW24K5izchjsmdoU3VIzHzuyNyf3i8xzIsowft67B4z8VY1+FHwsuG5R0nRXeIB74YgPmn94bsiyjw51KIi2WARhROX6i1R53vy+TlHvZwXx3frJ2Nead0h2frz+gTytIt6FNmrKdBVfEr/OJc9MRjsiYvq8KZ76wLOF2c9MrwFmcsNqjim+qy4kbJ7QDAPx0axqGzv8OvNWJomrl/p+d5sLq2f1w4SsrgaJamB0ueDwpePSrv+JGVsVSq3aUdMt2gOdZTOiZhS/+jAah5bjNGPf8b/hl9mjc9NFqdPLY8c2mCgQiMpbtCSDN5cA7a0pwQoc0/LS9GpVBxerRYjMea/Ne5d2jbVY6PJ409bn3G77ZUo0zT1Dq999lO2AWWJx0nOIRbzLthMRyWHb7KIx67Af4QhG4nco9MSszExzLIiMjQxdlASBDltE3bwfuPbUHPB5X0v1ubiaYnXjgmx0orgljV0UAMDt1v2VXSpr+7Hni67+wr8qPh6f2QjCi5I/ZW+FDilXE8e1SMGtiV8xc8KueCPGGDzfr20hLz0BRdRAsAxyImPHGsp3o19aNnOxMCPYAgLVYdIuSuG/ckz9ixgnZmD2xm/586Mjb8d47yrOyboAioLyL/D7HA5vIGTSMnOzE7wr3nZ6OHnO/woqd1Vg+60RkOEx62xvevQ1+3lGDt36vxI1j608o3RARCbjt0y34dOYQpNtFPPPdFsw7pcffWufRgNnceCuZFrUvSE9PB8dxehSaRnFxMbKy4o3fASArK6ve8tr/GypTt2clHA6jrKws6XZNJhOcTqfhD1DsFo6FP4ZhVC9EBlaRRyAsIxyRYeL5Jq/LbTXhxrFdmqVe43tm4/LhHfDF9cPQLs2GPRV+vP/LHrAMo9Yb8IVk2E2CcV9YxYMy1SbCH5Ebta3aYAQ5Lovu3+cL179cICwj02nGjgNeeIMRvQ4Agz0VfnTNcoBlWTgsIrzBCIJhGRZRyywtQOA5ZLutddYpwSI27ZhbRR7bD9Qi223RH2qx8y0ij/01QTgtYtyyNrMAs8grGbVNibcLKP6AJp6Dx2WOm5ebYkV+mg0cx8Fu5lHmDaJ9ul3xtYxI+HJtMbyhCIZ3ygADxrB8KCJjRGflsV6pek5pvrxb93vx2Fd/wR+WkW5XssmmWkUEVJ9JX1jS19M504GLh7SHNxgBy7LgWSW5kOEah5KAyR+WkWIVVHN/DgADzW40LCmJJmoDEX05f0hCis2kJ+zyBiOwmQRYRB7BiAwGjP7x51K9fHlOSbSkJUESeQ5WkUelX/HoZFkWZoFXPTuj1+5n1w1FrzbuhOdB8zQ1CxxCkoyIrKxXm68Nd0+0rElQrjtB9TFgWVavl75+dZ7DohwbnmPBcywiMT3YDrMAgWMh8hxe+GErwCg+sIq3rSLcWMToOnk18RkArNtXDZ5jkZuqiH6aPy2vluXU61/kown9WFYpYxU5ROT4+3EgEi2X7NqtUSNNgjGetJx6LLV7CMMgbjklaZ2kJjZh9A9JrU48pyRHk6Ek/8hymZUkV+q5CkkyBI7TfYu19VpFXrk+AXBs9EOX5zgI6nVj4jlIsnKM0myinqRJa++1AcWDWUvWpfmWmlRhWFTtDrTEXoqvKzCsUzp++McoSDIgqddpWE2kFgzLELnoueM4ZdlsNTstxyp1MgksIrJyfMwiD4vIIyLJkMBA5Dg90RejRo6ZeFZPBFX3/BRVBfT7dezfha+sxN7K6AgVGQzKfSGkOUwwizz+8/N2LN9WjkBYUtqT+uK5tzKAi19bFeNLFsE61TtSW3e6w4Q1uyrAsiwKMmzYWebDlSM6xNwjAI7jcKAmiEpfCCzLYtaErrhoSHuwLIvhnT3omu1q0j069s9tFbGv0g+nRcSCywbj3EFtUe4NwWaO3gesIo/Xl+2ES03syLIMIrKSRES7pwNKdEVQy8w7qRtmnNAOQUlGnvrx77RE11lSHQDDQB9CWOYNwWEW9GPlD8m45b3fsXBtsaG+3pBk8GwF1HOq3jMrfWG41GeLGFPOaRHAMNDLBcISUm0iiqsDcFmFRh2ra0/siI4eByRZxnd/lWJXuQ+lNUFsf2hS3PMpdrmls05Ej1y3wUfaYRHx2LS+OGdQPuyql21NIAyLqXF1YVkWj5zZByzL4uOZQ5p0zjNdZhRV+Y33c55D50w7jmvrjkuAGYtWv833TwAfc7/u6HHgsuEdGtw2x3H4fc5YsCyLDKcZpTVBfLBmL55atBl7Kv24b0pPPDS1N8Z0y9Tvw9r5im2b3pCE577fCiDxMybhH8NAFDjwHIOt+2vx+LQ+eP+qEzCko5KobENRteE8atcWAD05TEl1EOv3VWHxLSOx8b6TGn2+PE5L3LQ26rMHAE7ooNgMPHBaL3x23TBwMfc+s8hjye0n4vYJXfHEN5vw/V/79WvJG1K8gSf2ykZIfbcsqgqge7YTnTx2w/Yem9YHAHTfVgDolhMVGLTEYp08dnScvTDpvtQEIwnvk43986l176F2thSk25BmM+kBAf3apuD7v0qxvlgdUROU9OdiQYbDcG8CgCtHdECa3Ww4ZtrfX8U1uPKNX3D/aT1RkOHA+J7Z+rzf546DSWj6N0Wm6tP5/a2jYBJ4pNlMKPeFDNu3mQVkuuLPeW6KDU617S+/YzS27feCZVlsP+BFbTCCP/dWYf0+5Tpsm2ZVE5IqXt7dc1z459l9MfW4XPRp48KSLWXgGAYsw6DKH4bDxGNc90yIPIv1RTWY8uyShPV/bekOfLuhVB/hAAB5qVbUBqPvmSu3l+Pr9SVIdyjnJFf9NuA4Dr/erbRft1VEms2EsCTr702H688ssOiapVxP3bOVazrDYU5YVnuX2HHAi90VvoRlghEZv+6uxHH3fYv3Vu8By7KY+pxiSXjLuM7494yB+HHzfrUNSnHLa0n1qmPe3Rv684clrC+qxqguGbjwhPa4ZlQHmAUWc0/ujhvGdkbXbGfSZU0CD5tZQLccF07qkZWwTKrNhHJvWH8HBgC3Lfod1ibVhtmTuuke94DyXsayrP6+uWpHOWqCitdssrp8sGYvxj35Iy4b1h4ndvWgtDqIB6f2wvzTFYuEDfeehBO7Kt9Z64uUNt4z14X9NUE8fU4/PPntZlhNPPrkuZHrtmDHAS++VofH3/Tu7/p2dpT5sHyrEjCRajPp09+6bDB4LlqfeZ+sQ03MeSiuCuCzP4rgtplQ2CENXbMckKHlO2AT3ts4jsOH1wxFrzYph/S6znJbcffJPfQo1wUrdmHbfuXfe6sCermQBHz2exFe/HE7pv9nJViWxdD532Hqc0vRMdOBfm1T8f5VSofjo2f20c+vyCvX3YjOGXjm3OMw6emf8dbKXQiqzxLtnY/jOHAch6JKP3LdVsOzX7tHfDJzaNL9cFpEw/3xwsL8pGWtJh7ds5W2rOkAHMfhnOPbom/bFIzskoGnF2/B73uqwLIsHl64scnH9Z1V0USLb67Yhf21IdjNjX/3Otr/GkuLirKiKKJ///749ttv9WmSJOHbb79FYWFhwmUKCwsN5QHg66+/1su3b98eWVlZhjJVVVVYvny5XqawsBAVFRVYvTqa5GLRokWQJAmDBiXvoTvmUcUwi+p1F5JkCHzL9Nw3lW7ZTtVrU/kA1UQHjmVQ5Q/Baor9aFTK+EMRpNpEeBOYkCeiyq/0su+vURJ4VceYZgfCEZz05A+G8oFwBO3SrHhv9W4laY76EF20oQSn/utnPeGK4tcYUfwkeRZXjCjQDbfrUjcTZ2NwWQVsKKrWk38tunmEYb5ZYHGgNqgnCUlEbSCcMBEYAGQ6TWAYBiaBhccR3+OT67bgATXjpc3Eo8YfBssyYFkG1f4wTDwLX1DJpls38FiWZfTMdeHsgXnYXxOAWWBVP0xZP8feYBhpNhPKaoNIsYl6YrLYjJ+hiIR+bd2oDUbwwvdbEidvUqf5QhG4rSK8wTA41XxOG+IZjEhIsYmoDoRRUuVHSbVfr4s/pPiK1gaVpG6K/6fy8ZKiJlvQzivHKh6f2ouVwDEwCywqvCFd4NBEydghbz1yXEkzhnIcg7Akq8KbpCbuipaVJOgf1HURYjxhY49Z7PLaPIFTRDReTfQVm9PHInIY0jENA/KVZBpapnte3a5yjUf3h1OvAQD4v1W7wDCMntBI5KKiovGYGU+eLkQnSPTlb0RWUG37sYnauEQXSAICYUWQisQl+lLqzbIMJEmJ7ohNuMGxDAKqx2vdc2I38WAYJWIp3SEiX03Ox8UkWxPVbQkci95tXIa2axE5VPvDEDklwRajJvoClHPnD0kQ+GjSPEAR9SRZ2YbDzCMsyYhEZLAsg0hE1hMRJUocpyVi0aKsOJbVrwkTz8KqdhKEIzIEnjUkgZJV8Vrzr63LCQ8t0rM4x/Ljpv3YVBy1zQlLMrwBxd9X29fiKr+SVDAmak7rUNP8t/ZV+GHiWIzvEY0QGN45HTNHdQSgJE+ISNEESnYzryetmNgrG6vUJBdXjOhQ76iJpuBWX7ptJg7pdhM8DhPKvUFDgkWLyOHrdcWoUa9ds8CiNhCG3aQmNeJYzDm5O4oq/brgHQhLMAksKr0h5KpCemxbapdmxX8vVt5/Um2KD7jIs+A5Bif1yEJtUPGpNtdJNNhzzpeGyMQbxnRChsOkP9/KaoN6so7YY6Ql8fKpInBATRC0fX9twudIIm4e1wUpNhEMw+htff1eo4HP6K4efX81smOSff3z7L4AYEjOZlPfFxpKFJiM3m3cTSo/qVc2bp8QHxn81Y0jMO+Unnj8rOhH28+3n6j/OycmCVJjMkknQxtN0T7dhnNfXo7lWw/gyW824Yr/robLIoBlGfy8eb/iEa4mvJz3yTql401l5fYyBMMSbv/gd+yt8OHGd35tcLuy6msdkZQEdpow0CPHhRcvMA4zvu7EjrhkaHv9XLosAvrkuTH347XYW+lH+3RbXOfAwfDulYW4ckQHtEmxYEq/XJw7qG3Ctp3jtiBHrcuW0hq8tWIXRJ5FltOC8togMuwmBEISwhEJ+yr9GNXVg69vMr57WUUe2x6ciGp/GO9fpXyfxF6rV47ogEfP7IPcFOP1W5fBD3xrSD7aVLQh8Me1TUG6XYRJ4JBqF/X3lo4eOxZvLMWTXyvDhSVZxqaSGnyzPn7Y8Pp7Tkp4LWtokWFtUqxx85z1JParj86ZDsye1E3/nWIVDO8ljSXDbtIT45z89E946ttNqPSFdDsGjQv/swLl3hBSrAJO7ZuLkV086JTpwF9F1WiTYgHHMjjgDcMflmASOPy4aT+m/2cFft9dmfAdnmMY7K8J6Ek926fbcHKfHP3dJCLJGKp2VNhNPL7/x0jERk1rzwyXVdA7uxIl+DqUbLh3gv5uMLRTOkw8q7/LJMMfklBSHTDkYjjhwW+xvyaAdfuqMPXZJYbyEVnGI2f0xswTO6FzpgNfrlWEwkSJpcprQ+iS6dB9WZdsjlrjXPfWmoT16X73l5j01E/6u4nbIoIBgxlD2uOUPjm4qRGRgXYTj+fr3Ms0Um0iyrxB7KnwIS/Vgg4ZNpzWr42hTK5bSZSlvdd0VJNRdslywGnmMX/hRvSZ9xUAGN6tYrnl3d8QkWR8/kcRUqwiymqDcFtFjO7mwY1jOsMscPozcZ/a2d1ffYc/uU8OWEb5DnnrssH6t1ym06y/P3y9Tjnuzy7ejHdWKaPIUmzRtlzYIQ1f/FmE9eqxt5l4jOkWfefSRleYBQ7Pn98fH88ciqtGdsDaeeMbOLqHB5dFQDAi4alz+uGpRZsx8SnF8mX0Y9/rGoRV5OALRfDwwg1gEM0vsrG4Wr/PZbnM+PHWUeibF+2IM3Eses39Cr5gxHDf76yOXjHXGQUSkeW4d2eb+s3YKdOOxjLv1J5J5zEMg3euGKzuV/Q99MGpvWA38fjH+K4Y3jkDU575Ge+s3ImPf9vb6O1qaNcQALy1Yide/nFrvQlzj1VaVJQFgJtuugkvvfQSXnvtNaxfvx5XXXUVamtrcdFFFwEApk+fjlmzZunlr7/+eixcuBCPPfYYNmzYgLlz52LVqlWYOXMmAOXiueGGG3Dffffh448/xh9//IHp06cjJycHU6ZMAQB069YNJ510Ei677DKsWLECP//8M2bOnImzzz4bOTk5Lb3LRy1Ktm6ombjVRF9NUPhbGoFj9SgfTfhxW0TsrfAZPmg1/CElwUOiB3giqv0heJxKxGNeitWQLbQ2EMGGmGy92vrT7CaM6RY/bCUsySiq0jLMK8mRKnwhWEQOsyZ0i7vxajx0ei9M6NW08UipVhE7Dnj1h25BhvFGbRY47K8JwG5K/kFTEwjrH251WX6HklTPxLHw1Bkadv3oTkiziXoGa5vI6eco3W7CltJauK0CSqsD9dpg2E08SqsDCT3sfEFFXC/3BpFqE3CgVnmpro3pXa72h5HpNMMbCOPBLzZgZ5k3bj0a/lAEbquA2kBEzzavfZx7g0oiJlmWccM7v2LeJ+v05UKSDIFl4Q2GYTPxEHkleYHAsfrDRXsYC6wi4moPU5saaVzhC+mih/YSVndYYDI4VcjTss/LdR7WYUlCsu92bVuxH/ahiLHTRRMP+Rjxre5wSqvA4Z9n98Pp/dvgwsJ8eEMR+IKRmGg7o0jGsQxq1M6NUEQCx0Bfp0ngEJEkXThkoIjidYc2RkXZeAElEK4/M61V5FClRlDECjBaHZK95OrrD0WURF8RSRfRtTpxjCJahyVJjZSPth+eZeBXRc66ArBV5MGxyjq6Zjnx/T9G6ceKV8VcgWchyUpSrfln9MHj06KijUXgUBMIQeQVUZtlGF2sMPEsfKGIfgy18xIV4xlwnNLpEVaPfSgiqaKsZBDtNbR7LceqL4gx+2MSOFhEDuGIkpBNUNetEZGUKOpgREoqhCe7P2vRwaK6PABVgI5eX75QxNB+SqqVEQra/ejxrzfiwhPa4YULokk/T+yaiZknKh8J/dqm4Lc54/R52gcNoAgVV4woSFi3v4MzJjM8oNyfD9QGDfuhtaH96r3OzHPqPVqLdGdw0ZD2iEgyaoNhFBakYWC7VJh4Dp/8vheVvqAqgDH6Nd7RY8eQjorlwMjOGTALLNLtJmxV79HeQASSrBzj2kAYn/+xD/d+qtz/fMEwCtJtmHFCO9wwprPyIfq94jd5oDaYsJNR+3DQMvcqSQRN2La/Nm6IcUOYBRa9cpUPm2Cd+8C/ZwxEp8x4KwlA7bT1hZBmE3WBDUCTt/93aZduw3mD8hPO69XGhcm9c7D5/gm459QeSLUqCTMfO7MPfrztxITLHCxt1Y9AzdMyFJF1Qd0XiuCiV1Yayn8T8yH1n5+24boTO2Lhn0VYvaMc/1tj9A297PVVeuKyYFjCxa+uhAwlYrrKH8KZA/J0cQkARnbx4IYxUSuEm8Z1weXDCzBW7UCp9IXQM8eJFdvL8OOto5rpCAAD26Xi9gld8eDU3phVj7gIKM9znmVwoFYRVj66ZgiGd07HtxtKYDXxCIQjuPnd3/DLzvK4jgENhmFw3ehO6J+fCoaBoYOtW7YDZ/SPijSxYlUsYUlGSVXjfY81ft68H8u3HsAN7/yKf53bD3NO7o7ld4zBu1cW4rJhBRA4FteM6oAU9bxU+UO47sSOmDagDXYc8OKiIe3i1tnQ+8qTZ/UFEO0Yaw5y3RZcOix6Lz5YmwdWfRc/58VlqA1G8NKP27CpWPGXn/PRn/h2QwkcZgHf/6X4D2vvdV2zHOjdxoW9lT5kuSxwWnjsrvBjVJcMbCyq0ofgpttFlFQbc5B89OseFFX5sWD5Tlz/9hrcc2oPLLxhGLpkOVATCEGWZRyoCSAvVTleDjOP/DQbEuG2iHCaBZxzfFtdED2cjOmWiW/Ujoj195yUtN4a6/ZVIRiWdPsdANhb6cfeCp/eeRdLltOMMwcolmq5bgueOqcfZk/qhsUbSvTnWkSS8fPm/bj2rV+Ql2qBNxhBIBzBuS8vR7U/BH8ogo9/25vwXbFtqhWTe2fr9ZncJ1vvxGsOctwWfPrbXsxfuBHHt0tDAr0edjOPN5btQCAs4c6J3TCuhzKa97aTuuK3OeOQahP1DojqOu9Kld4QFqv33FP65OCz64bCooqFAJBmN+F69R6bpor4RZU+3DCmE07unaPbSNhMPMIRJfhEe3fMcZv198fLXl+Fbftr9d8TemYhzWZ8hj55Vl8s2aKMCBjUPhUvTY8K1c4Y8U3kWX079QUKHU60wJGOMd/Sg9orVmAjH/0OgBJkcu+pSrtvm2bV3xuzXWbDd0VeqhUdPQ68NH0A8tOs6KgKqSf3zUGvXBcm9MyCWWDx8OnRxG9vXhoNHuQYJk6UZRgGS2edmFRHOBgcZgHbHpyYMHgCUL25Adz2/h/YV+nXxenG0PGOz/XrVOPDX/fCTaJsHC2uuJ111ll49NFHcffdd6Nv37749ddfsXDhQt0HY+fOndi3L9oDfMIJJ2DBggV48cUX0adPH7z33nv48MMP0bNnVOW/9dZbce211+Lyyy/HwIEDUVNTg4ULFxp8G95880107doVo0ePxsSJEzF06FC8+OKLLb27RzXXndgBlw/voPcAaR/ZRwrK0HHloaRF7ritAvZW+BKKeXqkbIKHfSJq/GFk2JVrqG2qVfdBBKBvN/bFwRgVGD1Ot52kvOiH69y0nvtuS4M30VP75iZ9uU+GJrRluxO/pJkFDhXekB5llQh/SNIjU5MxrkcWercxevvcOLaz4QXZZuL1DNKZThM2FlXBbVEM693W+O1rOo3VxONAjRLNq0UaaJGN1YEwUu2KWJJiFbG/JgieZfSPfUAbGivogvCB2mDSa9cfisBlEVATCOsPIE2wUyIQFfGtQo2U0M5tOCIpkbKBMGwiBxPP6ZHA6XYT/rpvAhyWaKRsMEbkspl4WEQOld6Qfg1oQ1Ybm0xFO1aagFYXTUhNhPbyExsJGVIjMjW0uvJqBGsiQTFW1HWYBQRCiiCpRIQCE5/60RBpxzGMLrp5Y8RbQNn/UETW68QwyjGuK8pKsiJEhxLsc6CBSFmbiY9Gyqov5SLHQqtGbKRuLNpuB8KSKuBr14WsL8dzSqTsac8uwW+7KgyirBYpq0W/xuIPRcAyTJwgzKlRtf5wRBkVIMu4eVxnJbKJ5/DUOf0AKEJztT8MkWf1BDMmISZSNhiJivAxkbLam70iJMuKvQejiDF2E6+IsnWO/YjOUT9ulmEgSbJB+DfzrGLREZH1DoPY215EliFyLILhxMcZQIMjGUReWb92TrT7VESS9XuwdiiLqwI4uU+O8uLPsdhfE8SwzukNrD+6Q3YTr3f2jO6WiVkTuiVb7KDRjrF2vZh4FtX+kH7OgOg94a7J3QEoL+Gf/LYP9gTipy8YwZkD2uD49oq38b4KP24Y0xn/mTEQTrOShFCDYRhMG9AGVhMHkWPRVfVndVtF1AaVxHK1gTB6zPkSV7/5C/790zYASodUlsuMuar4UBuI4JWftwMADtQEEkbAacf1L9W31BuIINUuoiLmHthYTuvXBv3zU5CjRp40li+uH4ZT+uZi9V1jDddfYUEa1t0zHivuGN2kerQkPMdiemE7WEQOP912Iib3yU7aZg6WTpkOXDVS8fn74GplaKUz5praV+nH4IJU/bcWKS6pnT83jeuCyX1ysKs82umpvRt9v7EUW9Wo993lXizaUILaQBhui6C/E8Qi8ixuGGOMRMt0mnHNqI54UI3W0q737BYQoDiWaTD62CJy+PCaIXjuuy0Y2SUD3bKdGNRe6djQnsPr9lbh9aU70D49uSClRdxte3ASAOCVGQMBRKOStLNc7g1i+dYD+O+yHfqyvmBEHVXS9KjuhxduwGNfKdGvbosScc6xDOwmXn/O/mN8V/0eX+4N4fzCfJzUMxuLbxmJ2ZO6N3mbp/bNxYD8lAajf/8OBek2nHN84vwHDfHthhKDnYQWPLGppAYTe2XrHTaSFB391jnTgemF7VDtDyPLZYLTLKC4OojBBWno6LHrQtrwThn6aA2Nbftrce8U5fu1sEM6zh+UDxOvWBXtLveh/azP8Y/3fteFq9iOi7r0a+tG21QrxvfIwnUnNt7buaUQeRYd1Qi/xgjlPXOVIdIV3pBh+p5yH8q9QWQ6o0Kf0glvXOcpfXLQPceJb9aXYLV6b3pj2Q6c9/JySLIi3NYGw/o73wX/XoEiNTL0zv/9GVefzpkOdPI4sG2/cj/Ldln0c9kc9GvrxlfrinHB4HzMP6M3Pp45JK6Mwyzg110VePWigbhsuLETmGEYPaIVgB7l/d3GEhRV+rF4Y4nekTasUzrcVhF92rhxZn9jNC4Q/Qb4ZWcFzAIHl1XACR2Ud6OPrhmCaQOj7emHf4xCMCwZRqKNevQ7/bvjmXOPi/teHNIxHY9+uRGyrLxbxo4+aOo37eFG63jSvWQtAoZ3zsDG+05CfpoVsiwj3W7CyX1y0D8/BT/8VYp7P12H+6b0hNMs6KJ4LGO7Z+L7f4zCRUPaAwAuGJwPhmFwy/guePPSwYb2o9n7AMBXN43AtAHx97rY0UDNRX2jweq+i8R2rDREWA1oAoD7T4tqeS3loX40c0jCIGfOnIkdO3YgEAhg+fLlBguB7777Dq+++qqh/JlnnomNGzciEAjgzz//xMSJEw3zGYbBPffcg6KiIvj9fnzzzTfo3Nn4cpeamooFCxaguroalZWV+M9//gO7vfGh3scio7tloqPHrg+3B+pvpIcagWd07x3to9VlFVBcFdBFiVi8oUiTImVrA2FdOMxLteoRdkBUjN1V7sXvuysAAP6wElWp+ThqXDWyA369eyzunxK9+WgPs4ays/4d2qYmHjqkHStbPZGy14zqiD5qdFkyeua66n1hVLYR/cDLdJqxsagaGQ6TPsy6LrHDoCvViCavPtxV+f8by3bAbRF0T8L9NQF4HCb9YxBQInM0SwJAeWAkEzujkbJhXbjVIlSq/UoEIgNlWK7bEt3fsGoXUBuIwCryMPEsqnyhGP9OFnb1I4tXh6FzuqClRLtV+IL6tao9kBorUEiy8opq4jlDRm29fqqnbCK0Xu3Y+XWjUvkYi4OIFC8obr5/gmGd2rI+NSJYj8qMWac2HH9AfgqyXWbDNWDiVR/cutG7XPSYxZat8IXihMxkkUUaNpHTO1e0Nmjio8PvlQjY5Mc/EFaulSpVBNU6DCRVANf2eev+WkP7Us5/BHxMGUAZmjy0UzpYhkGkzr7wLAOeU8RcbVsdPQ79mJ3SRxnlYRY4VKvD3UKqBYEWPSrwiqirjXDQjq3mfcswyjmRpKjI6QtJsJt5w/Wq8drFx+v/5lhG9RdmoFnEmAROiSSWZMWjOEa4BqBHc4el+HVrPe21DXSaKZ600Y8Drb2FIpIiyorR66ekKgCnRcAt47voIrY9wSiKZMRGmbQ0tphIWcUGJTYCWdmnwaoAFIxIeHjhhrjIEpNqTaM9V0SOVfzFVRHLbRVRrkZwaKdl/hl9wICByHO6EJBqE+ANhmERWZR741+2WcZ4n4rtEDtQE4TTEn+MtfIz1I/G2mBY8fJu5PM4Fs1702bidb/cxtA505FwiBzDKD7MmqfmkUaWy9wsQ/UTceWIDlh4wzAc11b52I+NYvrn2X2x4NLB2Hz/BPxnhhJdLssy5ny8Ft//VQpAiSIqqvTrHXrd7/4SgDoiCMCPm0p1Ib42GMZ1ozvh3xdGI9UbwmkW9I/QMd0yMWtC179l3fB30dqcFgmVl2rBST2yYBN5/LqrAptKFJuVpgwnH9XVg8W3jNR/a3fMstogPvx1D+768E/sq/ThqW83odvdC2EVOYPvZCK0DpRYNpfUYMV2JfN2mr3++uW4zGAZIF19V2ifbjvoToH3rjqhxa5fQOlceHBq74YLJkC7bv97ifJs057pIs/iX+f00zvLEnV8p1gFuK0inGYe1YEIzhvUFs+cG00m27etG3srjJGytQFlJMO0AW1QVhswCBGabcLybQf0+1QyGzFAGWreVrUIONoEjYuGtNP3sdwbTWpoFTnsqfCh3BtChsOEHJcZ5xyfh58379dHxcVSWJCGi4a0w44DXsiybBhB1SbFCm8goouyW0pr9FFzdY+W0lEko126VbdsaW4EjsULF/TH3FN6qNZR8c8ibfTL4IK0hOuYcUI7DO2YjhO7ejD6se/xzOLNmPHKSnz6+16sL4pa+WgJMHvmuvBIjIephtYxtmhDvCVJQYbdEFTQNs2qR6Y/cFo0cWG5N4g+bVwJr70UqyJG/lVcE/cO1SbFckR1gDaEFliidVjmui3wOEww8RysIofVO8ohcAzcVhHvXlGIkuoAvvizCOO6Z+KxaX1w2bCCpOuu+13aIcNuEN7rkuu2NHo0ZUuiXZ8n9cjC9MJ87KnwYfaHf8SVe3P5Dr1javv+WrS7/TMAyjvNkI5pOG9QPn74xyg4zHzC79hjnSNnbDpxxGAROfiCTf9wamkEltU/6LSbpk3kUe4NJnwBrPGHke0yN/ojsDpmCH+bFIv+YAegP2QWLN+J815aDkCJ0jMLHD77fR9+3LTfsC63VTQImNoLfX1D+P8O39w0Ium6tWPlqCdS9qqRHZK+FDSF2IiWLKcZtar1gBbRyDKMwXNL+5fIsagJhJFqE/UPfu2lOFYsd1tFlFQF4HGaURuIYOGfRXhv9W4EIxJcFkEXdCu9IYM3TixRT1lFTBS4aDRnlU+JiPxhUymKqvzqB5lSy5A63NsXUj1lBRZV/rDh2mNZBj2zbOA4Rvfo/OfZfdEp0xHnKavR2Aeu1vtsEuqLlE38op5iFXDZsPYGO5KQGvmrETvUXRN4Y1++6n4Yi7ySJEvz2tU6LmI7MzRRNs0uIhQx1k8bfp/M51bgov6kIs/i6jd/wS87Kwx12HYg6gV3+nNLsKuObYWJj3YwaS+lJiEqrgbD9Y8GkGUlmikaQR2NlOUYRq97rN+nst+qHQBnFLZz3RYIHKtEndYZ/aMJuIFwRI/WToRV5FCjisThiKQK9VHv4lhhO9YnWNsfTSTV8IcisJmUF6T6joXuKcsw+rVm5llYVfsCzQNXlmHw++NUi4S6l6YvFAHDwNC5Isuy3gOvDZcyC5xhyLp2PENapKwQ9aouqfbDaebhcZhxXFs3gKZ9vKbaovYFLcmWBybqoqWJj38N0xKOafO0Z1hdixmnWUBJlT8m0RuLfZV+fah+qk0wfABr2M082qRY9I/kNJuS3NAicKhIUJ4BY6hn7OiT0pqA4Vl3x0RlpIhFTZB3fPtUHN8+FYGwBJuJNwi6TWF8jyxcParDQS1LRHFZBD1BDxD1Qb/n1B44tW8uWDWC9MSumRjTLRPfri/Bf5ft0O0orCKHpVsOoHOW3XCtWETF9urDNXvx/V+lmDmqIy4f3gFpdlPSrPTJ0Nr4wHapuGLE4T3nHocJU/rm6D6qVlHxkEyzi/hx0350y3bi8+uGNTmAITayVutcrPCGUJBuh8ixKHxwER5XPV7dVgGVCdqlRoU3iHs/XWcYUhoIK8lVtQQuyTrtNV66cACyXOajTuxrKhvvOwkr7hiN/Jikb/sq/fhuYykYhtE7/RIFcyy/YwxGds6A0yLoIrl23i8a0g6dPA7sq1TECm247m+7KxUfecSPPPIGI7jn1B5KMlmriL/um9Bqj/+ck3vogtRHv+5FSbUfsiwj123Btv212F3u1aO50+0mvLl8JyYlsHJjGAajunjw0MINWLrlAAJhCaf1ywWg5B659f3fce1bvwBQRr5N/88K/GfGAMM78+odZeh7j+LT2pDlwt9lfI+sejs3tO+BZMEZQzqm441LB+mjFD//Yx965jpx32fr8faKXXq5hnzOx/bIxBfXD8OZ/dsY3tGTMaFnFn67exzOGpini4bfbSzFRzOHJiyvfR/M+fhPPTpZg2GYI7YDNBHa96o2pP+jmUN0G43j2qZgzsdrcf5gxY6IZRl8/4+RABRbpJ65rno7jiMHMeLhSOCiIe3RI8eJDIcJ7dJsmPz0T3hj2U58va4YZzy3BJ3v/ALr9lbhzv/9ie82lkKSZN3qAQCmF+bjzUsV39q2aVYsvmUkzhrY9jDtzZELibJEHNqH/ZFGrPCjPQAsIqcKXQkiZYNK4q6GIrE0agNhPSrCKvIGM2vNM7LCG0REluFTfYtMvJLVMDYkPxFr1eQkzekBE4s2hCgR2od8fZGyzUW/tilYd49i3q758yoRNFFx7X9r9mBXmVcRBdWXFZFnUe0PGxJ5+UMRnNYvF+XeqLdXqlXAX8XV6JrlQG0gjEUbivH60u2QZdkg4lX4gnFRyZpQ5AtK+voEjoXTLGC/OixIi5S9XR22HBuhF45EozrNgjL8t+6wYwB4+eyuuhjJsSxO7ZuLVJsIt1XAngRWG421L5ChRDeKXGJRVkpgCK/BcyzunNTdIMJKMoyRsrqnLKuLbskibwEtUZmsR8pq5y1W1NJsHESeMwhz/zq3H7JcZoTrJICKFYptJl6/D2n1jI3CLKr0Y/7Cjfo6V+8ox/YYkVaro69O5HWazYSQpNQlqPqpJoNhNLuAEASe0T96tWPNqh9Zlb6QwbOZ5xTjf14V/evCsYjzZOI45Xj7Q5IeRZwIJWoqGinLMIweLaJFAcUmQlSOH4M0Ncqcq9MxsvOAFwLLqp7iyc+3WeDUCGFVtOeUUQJmQamHluirbr1ZhkEoLMeJ+t5gBClW0WBBsbmkBn3mfYXj26fiE3Won1lg9WRWsYTCkuIpK3C6gF0W42/qPAi/qkm9s/H25YObvFxTiT3OiZ4JwXAEIh/1dNZHrtQp57QIKKr0GyJlY9fptAjYccBriIwGlOHU/zq3n96uHGYe3mAYVrWTsy4Cx2BYpwz9tyYScyyDokq/7lcHAJcP74Arhhege44TVlFp9+3SrAhFJNhNfNLOhoYo7JAWlySF+Hu8dvHx+iiK6YXt4uYPLkjFV+uKAAAr7lS85c0ih1L1PtL3nq8BKJ0pFoHDLzvKUVLtx8rt5Tilb44e3X8w/HLX2INetjmxmXg8eXY/faivhtbmJElG9xxnokUbzcB2qeifn4Jyb1B5d1GfJb/dPQ7t021om2pLGMGuoVmJ+EIR/blS4Q0hxSYgx23GyC4ZDfo3Os1CiwyJPdLQBKK8VAs+vGYIFt8yEml2EQPbKcLTpF7ZSTvnlI5oRTQM1nkmzTm5B3LcZuyt8OOLP4qwYlsZymuDWLGtDG6rgK7ZDgxsl2pY5sGpvXCSOlw+y2lu0ELsaGd45wwc3z4V767ahbV7quANRtAhw443l+/EC99vRYpNRCgioVeuC1+vK8aFJ7RLuJ40u4jS6gBKawKoCYRx5YgO+PbmEeindsT+uacK5w+OCj7t0+0Gkf1ATRDeYARVvjDap9twYWFiv+9DQV6qFRvuPanBclqQULbLjEyHGYtuHtGkIeROs4Bu2U48cmYf/GN8lwbLMwwDl1UAxzIGj9P6OOf4PCzbWoYnVF/po5XOWQ49QphhGMN3Ul6qFVtKa3B8+2hbzk+zYdP9ExrVMdc5xkLoaKPaH4bdzCM9xpP/7RU7sWpHOdLsIqb/ZwUAYGeZ12D/eOWIDrhkaHvDutLtplZ/vzsY6IgQcShRXEeiKMvCq0YUaGKZVVSHhMc0bpZh9Pk2E9/oSFnN6Hz9PSdhcEGqwRvKF4wg3S5id7kPKVYR3e5eiBXbymASOMw7tWfSJB4auW4LNtUZ/n2o0IbVHaohgFrUVqbaM6pFswGKWHDLu7/h+79KlehCVUARVV/FVKuon69AWEK2y4z91QHdRiDHbcHavZUoyLChNhjRh0xrl6v2/zLVnzYR/nBENxjnWOXFo6Q6oAxJ84chcCwuGdoeX1w/TBU/o56ymsAmcKxyPQYjCR8sPMsiEDImTuqR44IsI658Y8VyWbMvENiESQuS+aMa62WcH+spqy1rEqIRrGw9LxkiHxXjeJbVI+B65kZ9hzlWSdKk+IpK+vom987RxV9jpKxs8NrVxMa6iasA6AJsu5hIhwM1xg8ps6D4UGser/+95HjkpVp0gb3KF06YJDAWq8jh9SU7IHKcbjmgHWutPpW+MKx1PWVVkTORZQmT4B7Lq+sLqsJ0MlHWLCgiseYzyDJKGz+hQ5ohyhiIWlIIHIs5J/fAreO7JIzEYdX6CvUkdhR5VrEvYBj1nDNok2JRo+5kPVJWkqMCoOJby6giuHG7vqBiDREIR1Bc5cfOA179+DrNvJ68ycTH+ym+eekghCVJtxTR/Lv31wT0DhcTz2Jir6b5wznNQrOMGGgKCTsUQxH9utl430l6NHHdzlKXRcDucp8ebW+qI/BaRR43vPMrBj/4rSF6WeBYQ8S+5qVrETiU18Z/6MkAzh0U/cjV7tFWQekgqCssz5rYDW1SrOjdxoV1e6sgqO2/pToliYNjROeMep8Z7dJsWLu3Ci9NH6A/T51mAVW+kCHaNhSRYRE5fLuhBD9u2o/NJTV/20fwcGeXbwiGYfDptUPjfCAPhpvHdcHMUR1R7g2h0hfSo95cVsWPt3OmHcu3HcBdH/6JJVuMo7KWbN6PZxZvxgkd0rC/JoCCOz7Hf5ftwKAHvkWm04x/jO+Ke06pP2gAAFJsIgoP8b3vcMIwDPrmudE+3YbXLjpet+rJS7Vixgnt6v1uGNoxDY+eEi+sZDhM6rD7IJ77bgsGPfAtLhnaHmaBw0VD2mP2ZKNH7znHt9UjCNu0oAfvkcL0wnb4x/guCEsyHl64AbvLfUiNsdW4emQHPHBaL5zQMR3pdlNcfgENrSNpf00Q1f4QHGYeHTLssJl4bH9oErY/NAn3TemF/6m+2do7tjaSSgtqWLG9DC6LUG9m+kNBY56Lt4zrgt/njsP+GsUCrU2KEo350TVD8Om1iaNXk9HUqH6zwDWqs1rzwD8SktD9HdLtJsP7TiyZThP8ISnORz/ZtVqXggy7nnfmaGNop3T0aeMyBKB8q9phnDUwD/trArhkaHsUVfrw6pLteplT+uQcUVaYRzIkyhJxaGLJkQbPMrqtgjYUWfuIjPWUFXlWH+5qM3FN9rCziBwcZsHwUeENRpDhMGFHmVf/WNhT4TMkNaqPn28/sdE37eaGYRisnj3mkG83zSZi+R2j4Vej2QDoYqgSZRrW/bNEjkV1IAyXJWqSHlATtVUHwnrUW9s0K7Yf8KreUWFwSToQ9lb64amTYVtP3qQm+gKUCDCXRUBxlR8uq6BGRCrnSctkry0XkmQ9mlSLxguGpYTWGdowdM4gyiaOpmms95qsCmKKp2yCITAy0NBzr+41aIyUVaOWtYjjBFk/YxFV4VATKEurA7hudCd9WA+gJJXSPFKDkbqRmKoPKW+0E9BEa4aJtnOtnrERk8Vqgo78NKseHVRXrNYiZbWkLNrwwVBEgsAy2FPha/BjyCLy2FhcrUYGK9O0iFFN4KzyhwydALwqAgscm/CFWzuXsbAMA7Og+AUnSgSmYRX5GPsCZT35aTYsuGxwvCgbE/3cLt0Wl6VekpU2xanRtvUNnTTxLCQpan/Asyw6ZNgxRvX7DEuSfk3ExnSyDNR6Kr+r1R50byiMVKsIf0jCg5+vx7xP1iZMCGTiuTgx0iwYp0Uk5bopqY4OpWcYBs+e1x9HOomuj5N6ZOHxaYovnInn9Cj0uu27c6YDq3aU6R8IaXWELE3YLalO7pnncZhgNSmRsmaBNUTKDshPwZn928SdF61zlOMYg6dfXfQEcmrEfCgiGRJLEUc2XbIcWLu3Cu3SosMxc1MskGRgSKd03Dy2MwYXpMIfjhjOa5pNPGIzazcnPXNdOCNBUp2DwW0VcNeHf6KoMoCHpvbSE7FV+UKwm3mc3DsH/122A8u2HDAs9+GvexCWZOS4LXj++60AgLkfrwWgePJ2yXLoPqT1YTfxuHFs5wbLtUZYljHYXU3unY27T06e5MxtFTG0wB033cxz2Fxao/8ORiTd67M+Ft8yEh0yko92a02kWAUIHIMNRdVYvLEEDjOPNWpUfJdMB8Z0z4TdxGNVPd8smj/yvZ+uw8e/7k2YABNQRu4Bis0dxzIYNn8xHvtqY4t5yLYkFpGD06x0wnb0OPT35D55bkMgREvhtgoN2u9ZRR4zTmgX9+3VmvA4FMG5JfPDHKk8cFovnNQzG6k2EzKdJswc1RHZLjM4lkGa3YSeuU7YRA4f/roXT36zCQBwYlfP3x5JcixBoiwRB8dCz859JMGxDLzBiMHv1SJyqPSFIHLRG6TIK1E5smzMpv138IUUUba0OgCHmYdZ9fUUD5PQ2lSa6unWHDAMg0ynWYnAU18gNG9YXvVxjY2UlWXlI18bFq750QLQRVRteJ2SZTWie3MmEiPrEyg1IYRjWbgsAoqq/HBZlIROJvWcmgQO5d6oDUI4Zmi9lqHXH4okTDKnJfqKjQJlGCYu++rorp7klayDqpPqAmPcfP0/yakrssYOrdf2jWUZ8CyrWgskX5f2UqhZHTx3fv+4YWCsGvmpDLWPj5aMSLIqsCn7E4xIuoAvy4gOy+eV5TTRtbQ6gH+89zvmn9EbZoFTIziNkYSyLCuCli7KRnTRURt+r0Rr1z/MPSpWR+0LtEhiTeQNhiVDgg6OZeBXRflE3sYRKd7rlGcZ1XMzkjARmIZF4BSPW1Xojl2N7iWr/l8739rxq0tIjWLmuYZHR4gcqyf6SpRUTjumEVnWrxuGUSNlY6xKes39CjWBsCFSNiIrSYG06zp2aKh2r43tadcirzUikqz6oYaQ0kAywiONRC/3BRl2nNg1U/9dEwjhutGd4qI3spxmhCKyniW4S5YD719VqM+P9atONlRsxZ1jYBOV5HEcyxqSdKTYRByXn2Lw9AWiIxI0MT8Z2vZ3lnnRJsWCCT2zsWTW0ZPw41hH88aL7cwpLEjD+1edgFP65ODa0Z3QPt0GfyhiiGxt04RkbISCdt/aU+FDtsusJ2K7f2ovDO2YjhT1+NZ9LmjPvCynGW+t2KmUUZ9Th0KsaY0UZNhxat/cJi/HsgxkWcmsrtEho2HP0vbptlbrJVuXdLsJ5w/Ox5yTu2P7/lo4zQJSbCJO65fb6GMgcCzeu1J5zlX5w/Um87xieAEsAqd/Qzy9aDPW7q3CHRO7HlbbgoOFY6N2df86t98h227XLCdW3tnws3vuKT1adVRktsuMV2YMbNX72BB989xYevtohCUZLouAiCQjzSbig6uG4PoxnTFzVEcAwOxJ3fSEu0TjaP1d2USTYRkmzoPuSEBQPWXfvnywPjzCKvKKv2KMMGbiWby1YhdkdX5jEouEIhL8daLsOJZRhThlCLT20RGRZPTIUYZlHss35sZSWJCm+/pq/rayrHyoa4KVFtmnRYF2y3Zi5fYyzFA9pTIcIhxmPpoNM8WC2kAYPMvECeOaKF8f2pBhnmXgtiq+jClWETWBMAQ+mmSnvDaILFUIDqsJlDbce5Iu6mpD8+sSO3w9lrpm/P+eMbDeesbSr20Kiqr8yjFKYF/QmCuxrr+pECPUxA5dFzgmoYgai9YRoomyiTKIKhFyEd3/1PjSrWzDLHD68HNAEXIm985GWW1QF+m060OLEN5UUo1gWMLAdqn4/q9SbNtfC0k2egArw+mVNuwwCzEiOaNbUcRG5m4sqlbvHTsN+xAVhqMRvdo+x576WAFMixDmWSahZ7AWqRsLyzKwiZy+7mRal0UV0EROqU/sPUj7t8BGr2/l/4kFuaCajIxlGMM5SATDRBN9KfYPdURZ1dIgWn9Z369gRAIXU/5ATUAVZUUEwhLKa4Owm3i93e6I8QY2qXYNsRGlAm9MVqaJspW+kB7FfrTQPceJ+WfUn0ncH5LQxm2Ji6rPVH277TH30f75Ua8zd4yvbn0JBa0mHgdqgrCIxij8Dhl2gz1MXZgGRFmtvhFZsTrgWCYuWRlxZPPb3eMMv80CZ7jXR0duMFhz11j0u/dr3Dim0yGu5dGP1mFd4Q0a7umaL+9vuysBALWBCMIRCRuLq2ETeX3kQbJoQeLQM70wHxcNaYd3V+8+ZiJgG4vbKmLOyT2wZmc5vvijCN3URHRN9SEd0C4Vk3pn47Pf99Ur5s6aqOSHiPXwX7J5P+4+uXvcEPSjgW9vHqknXZzc++A9uw8G+t5VbABHNSGYprXCsgyuH90JFw1phxMeWmSw5rtlfBfc0gjfYiKeoyPMjzikcGzyKK3DCccqHp4uqwCH+jDVooxifflMPIeHF27AzjKvGinbsCj76s/b8fNm47CwdIcJ+1WPygpvCJN752DFnaMRjEjIT7MeksRZrYFrR3cyZC4GlMjjK/67Gn/uUT40tPN3zvF5+OL6YRjUPhU/bdqve0cVpNvxx9zx+ktBqlWENxiBPxwfqZrRiKhgXheBWaTbTdhxwIsUq6jYF8TMK/eG9BcgzU8zVhwKhqV6ImUjScWwg2Fop3Tcf1ov2ExcYoN/psFAWdSVbg1ezDEvtoGwMvS+XvuCmEjZZAnBWDYqfIbCxqhOVrUnsAgcQjEiulXk8K9zj0P3bGdcAitNpNUi5TOdJvAsg+vf/hUADMJiRI4mpTIJisevso8yQqoPbjAm2n3aC0vx3Hdb8MIPWw37oF0PjBqV7QtGEJYkQ6IvwOjXy3GaNywLi8hh433GRA4hSTII4hrakF9lTUkiZdXrUfOdTSSc83UiZhNtC1DFYY7VxdRkHKcm0ACUSI3YyFcNTYxVkqDFenwrgm1s5umaQBi+UAQpVgHV/rC+T5oou+jmkXpZk5oE0BLTzniWrWNfIOvrOFw2MQeLiecwTc3sm4zzBrVFnzx33HSP04yVd45J+lHap40bCy4bhHZp1nq9k20ih5LqAOwmTo+KnX96b9wyrnPCCNtBapILJTo9uSibZhfRp40LHBMf4UccHWhR2MkwCxx+310JSZaRYhOxdNaJGNmFPlqbitsqYv7pvbGvTuZyDS3SzxsMo8tdCzHpqZ8w8tHvsGZnBVbPHqO/pwDAHRO74rwknohEy+O2irp3pPsoG7lxqGifbsOK7WV/q5PukQY6M2OJTfy5t9J/VAqygBLYQuIocSRgETlkOs1YdecYnNSzafkbiMQcXV8vxCGBZZm4zOBHAgLLwBeKGCITtSi02A9H7d++YCShp2w4wUdkSIqf5nGYUFKtvCCX1QaR67bA4zDrYkRjvKIII2b1w8GnRs5qQpB2zqwij3bpNnTMtCMsychLteDmsZ0NosO7Vxbq4lqwjoUExzLo4LHjm5tGNKo+TouAdmk27Knw6QKRGBO1G2tfoPh8GpeX5MSesFpiKY5r/pennrku/L67Mq4uDJDUh1SjbsKkZC/EgwrS0CPHZRDS6qJF3WoCZSI4JibRV0QyrE/gWfg1UTamTVqEqKWFJrxpomJYklDlD8EfimBYp3RYVa8wTVyOFeokSTlnEUnW7Qu07YclJUI0oForAEClLwSnJf54TC/MV/1UZUiyjOMf+AZLtxwwJPoCECfQ+kMR/V7RWN9gPZIcQLLYZ0uM9YYkA4kOfV1P2VA48fBzE8/BF1JsHeoTZT+4Omq7oR3TuokDFZ9ZRhVgYSiv+dVq12dtIAKvGim7s8yLHHXkQyAcwfwzehuOq0lNAhgbcVzXvkBGfBK71sT9p/VK+rzJqMe/jWUZnNAhXRk1Uk8nolXkUVodMFhtpNpE8BybUJR954pC5KVaYBLYekXZge1S8dHMoWibakX2UZ78g0iMReDwwOfrsX2/Et2uWQwRTWds90zcPTmxl6nIaUklQ4Z79YHaINLsJlhFHsM7Z+DFC/rjgsHtcL+aPZw4tNwxsStSGujIIKCL1YmCGhpLolFIybh7cneDrQRBEM1Dik1sMMk00ThovAsRB882PJT1cBD1R4w+xKORssYPdg2byKMmxlO2JhBGzzlfYvtDkwzrznKa8fDpxpdYj8OM4irFEH7b/lrdMoFR7R3+fWHjh54TCmZNMA9FYDfxelKouh/+WrSryyLg2tHGoZAD22nDcxVhNlY3FHkWNYGw7rnUEEqkrPJy6LaKqPaHdQFQ4BhUeEOwqkIZyyCh72ai7OmJPGWbC4FjUeENxoloTALRuC5WgTeIOHWjClfcoXhGva5mIl621Rg9XrceQHTYeiK0hFeaeBPbwy+wqq+wiUeojn2Btn5NeNPadLU/jN5zv8JDU3vpL9g8y+gxpQb7AjVSNqJ6y2p2EpwavaslH4q99jQ/TW04P8sox1VQvVIjkoxqfxgmno0TZWOvQ45lE9pXRPedNSQt07DF2Hsku3S0DxFe7ZhIFCWpCeYsA3x78wi0TeDxKMvAHRO7ISxJ+G1XRaOTO2rWBIkiZXnVUiG2TiyriLJuXsTmEiUJSq0eKStia2kNBrVPxa5yH3yhSJzHaobDhGp/WO/QAZQI4HJvULcq0KwLiMTUBiKwCslf90SeRaU3qEfbPX9+fz2BWzIv2o+uGYpPf9+Luz9a2+D255zc4yBqTRwNmAUWpdWBON9houmk2ERcPLR9wnkWkUe3bKfe8XjB4HxcNqxADxwY0SUDwztn1NtJQ7Q8lw/vcLircNRQ9zusqTAME5fcMhksy2D7gVoUpNtwjep5SRAEcSRBkbJEHBxTf9TU4UIfihsbKZvAvkCLCEqxCXpEpcaecl/CdftCkbjMkh6HCau2l2H2h3+AYaLbVcS5xFmzifrRPii8wQgGF6RhaMd0AIjzZWUYBtsenFjvMJ1lWw9gzc4KgxApcvVHbsVy+nFtwDCMHvHXKdOOan9IrwvDMGAY6EIFm8TWI5GNBafaF7RU7+EvOyvi/GnZRtgXtE2zYuWdSlZbh4k3DOkClOHQsdRXf+1QhNRh+onQBDmRU6I6Y9fnMAso94ZgEePtC5T1yzj5Xz8BUNqeQ80QDyjtVWv7HMvCqrZFg32B6n8qy9HkaJqQ6g8pdY4VZXmWwQHVroRXPXW1eTWBMFwWQRfly70hcIzRviB23zT7iroevhqCGjlcF7t6LUlyck9v7V6niaOJyvEx13CHDHuccKst06uNC/3apoDj2CaJsiFJiveUlaIiuHbcNXE5FJFhFljsrfSjS6ZDT/SVYhWw/YAXOW6L3gkSO8z+woFZyE+1osofhjmm403gWBRV+pGq2pvkp1nRJoWSCyVD8eSt/3WvNhjROwVi/WfrRllrpNrEei0RYmFZ5phJZHOsEZZkEmQPASM6Z+Dz64Zi0YYSAMC9U3qibZoVA9RO6nS7iQRZ4phj9V1jG1325D45uP+0Xji9f5sWrBFBEMTBQaIsEQfLNpyJ+3CgDcWNFWW1D/VYUXZ4pwwAwILLBsetY0+FF0D8MG9fMBKXCCXTaca7q3fjjWU7sbU0mnhmRGcPerehrLYHw8ReWVh8y0j4QxHIcjSiLlE0VkO+SQ+f3hsHaoOGaYrYFT23y7YewCNfbogpEZ332LQ++r9/vXssMuwmPau9Rq9cFyyq8BAISQmHoVsTCBM8x7ZYpCwAjOnmwS3jjEbqDBqOlI3lj3njkeuuf6hpoihgDe0ekSzZGRCNGBZ5TUCNzstwmLC3wgeLwBtsFbRI0NhzK3AM3DYBXtX2wh+S9E4RnmXgsgh48YL+hvXEXl8mgdVFciWKNRKNlFUrZRY47K8JqPVkDef7wam9MKqLB4GwBIZRBV8uGinrsghxAq0/JBm8VWPplGnH8XrEdxSXRYn6kKGcz0Ro+6T4tyKhp2zdJGKJMET2Mo33EWfVRF91902Lnq0rvivlFZuICm8QmS6zHimr+VXmuC1wmgUUVfkNkbJXDclVEpv5Q4b7s8AxKK0O6MNE7z21J56mLK9J6ZBhx4ai6nrLeINhWE08nj3vOEMiJy16PBE0ZI0YXJAGAEmH3RPNh/ZOVJBhO8w1IYijj2kD8lDYIe1wV4MgCCIhZF9AxFFf5u/DSSKBSxMoYiP1OnrsePTMPnFG7qu2l2FPuQ+pNhFVvrAhgYU/FInzJ8pwmFCmCkP/mTFAn37VSBqedLAwDINMp0kXvzQOJjlPmmo7wDDRqE2BZwxD2F/6YSu+3VCCf4xXEo3JcmKx120V1SzvYUNdjmubootEiYZWA4mFCU71FG0p0eLlJNYZciNSfTWF+jy7dFG2jgVALLGJvgDjse+V68LjlYoIFwzH2xdoUasAkJdqxeXDO2B3mdKp4gtF9A4ZjmXgD0dgFXndEmDb/lo4zbxuAWDiOQTDEliGUf1elUjPYCSi103kWX0IvCTLShI5PQFdW4QiEr5eV6zXiWcZff2/zRmHJVv2G+Zpwm8iJvbKxsRe2frvpbNOBKDcc+45tQf2VweSRspqmHgWkiwntDngORbr7zkpfkYSeLbxgr6WPK2u8KtFykqqfYFW/2ikLIfy2iCynCbUBMLwBiP6PTrXbYHDLKC40q9Ha0brpiT6apdmjJStikkQxnMsnGYG/7v6hEbv87HEA1N7NRgJHYrIsImc4boEFMF8Uu/shMsciZ23xKGlR44T3bKdSYfdE83Paxcdf7irQBAEQRBEM0KRskQcWgbtIw1rI7N0mgUOZ8QMTxF5Fv5QBGc8vxS7K3zom+dGaY0xw23scGgNtyra/t8VhejooaRezYVV5PVEXxp1j31j0AQdholer7HWGxzLoG2aFQXpNt3SIBCWkg7jNQvKkPLYIecXD2mPkV2UyOsrR3RAv5gs9ABwxfACZDnjE9howt+htLiIFaebi/rqn243oXOmHZKc3HeSY5RIWX3IfYzSqCUu4jlGj3Cd0DNLHxIde54EjkVnjx21qn1BwGBfoAjgFlFJDOYNhjHq0e90T1nAaF/AMpq1gNG+gGMZVPnCYBhl/YE6569uRCrLKFGhsfuq/1v10q07xD8Zsclxphe2UyNl60fkWaWToU7Jp8/pB6eZb1KbasrQci3RV90kduGIBJ5lEa7jN8uqyd7MvNLpkW43KYm+AmG9kyPDYYLDzMdFygJKVGzdRF8Cx6pD8qPTGIZBv7YpIOLpm+c2RL8mIyWBP9/Adql45tzjEpY/mM40onVhFjh8cf2ww12NY4Y/541HXgKPcIIgCIIgjl7ojZqIg1OHpx5paEld6tJQdFSqTUS5V4m6e+H7reifn4LSauOwd28wPhGK9sGfLEM9cfDUvbpy3RZsvn9Ck9ahRTpLUnQodViSdSGM51hU+cLIdivDpQElIjqZ0KgNVY8diu+yCro9we0TuqJrltOwzKyJ3RJ+IGli4KEULbSkWc1JMrEVAHrmuvDVjSMAGAXJWDhWEeS09cQKm5mqmC3E+AA/d35/XUx85Mw+GKUK4oByPr1q0j5fKKKLtrECuD8cQY1fOdeSFN2eiVfsCHiWiVoqcCy8wYh+vgWWQaUvhDSbiEBYUrxyDeKicd941ui9HSts8tzfi5ROFtEdi4nnICfwnj25T05Sj99kNKU4p/oEC3XsCwyRsoxRlAUUCwlvKAK3VUBtMIzaoHK9LrxhGDiWgVXkcKAmGB8pyymRsiaDSM+oHSzk692cpFgblzRFo1u2E8e3j7fhIAiiZaD3UYIgCIJofZAoS8ShJWs50kgmyjYUHZVqFXGgJqhHQOa6LSitM3zeH4rALMY3h95tXOjgIf+u5kbkWITqDKdtqpCU67bgz3nj1YhIZVokRqAVORZV/hDS7cpwacAo5tVFm16fENlYWspLtj5sIo+81Po9YpuKVeSSJquKJZl+yNWxL4jV8TiWwdjumRA4Rk8AF4vAsQZxT+AY3VPWF4zajeiRsgKHBct34o7//QkAhkhZs8AhEI6AZRlwnFIns8ApoqyW6Itj4QtFkGIV4Q9F4kJV64qkXF1RNtajldU8hQ/uWpKRPNGXhsizST1lG1x/ncOdzPs2ERyrtTPjdjVP2bAkGT1l1VWbeQ6+YARuq4iaQBg1ASWxlNbRYeJZlHuDSSJlw3FRsQAaTF5FNJ4Flw5Ct+ymjQjp6LHj/64obKEaEQRBEARBEETrh75oiDhYRol2OtJIs5nwxiWDmrxcqk1EWW1Qj8zzOEy455O1hjKxIk8sH88cmjC5E/H3EHkWwXDyBDKNxW7iVTFIuZWdPzgft5+k+MfyLIMqnyLKVvvDYBhFjEt2PrXpzRHdejgS4Hw0cwjO7J/XrOt0mAVsun9ig+WS+VXqoiwXb18AAC9NH2CIlE2EJiDyLKvbF8TajWiRr5pot3V/DQAYIjZ5Tk1AxTDg1OvAqiaQ0uqmRVgroqyEQEiCUI9AzzBGUTZWtNXq1Fj7grpwbP3HBKjfU7YhfKGIQZhNFumcCC3RV12xPqxaGiRK9AUoAqo3GIbbIiiJvoJhwz3XJHAoqw3GJc7TznuipHNmujc3Gyd0TKdnHUEQBEEQBEEcYkiUJeI4UiNlOZbB0E7pTV4uxSaipDqge38WdkhDnzZuSJKsiyq+BIm+iJbj990VWLa1rNnWp3nK9sx16dlVBV4Rc5TEbiFwjDrsOonoqkXdNcd1cDgiZc0C1yRv0ObimXOPw6D2iTPacmrCK02kS1Q/nmOT2qUIMRHVAsfoXsS1gXBMoi/WkKivSk3WFRvNqQl9LKvVSYLdxKPKF9YjZbXh+GaRgwwZXbIcyHSY4ur01mWD9X/HirJcgiH7B3sd2EROjwpOhpac62DO+ZItB/DthhL9d1M6ETg9GjbajmQZiKh2D5Ek9gVaZHKKTcRHv+7Fgdqgoe4mNfK3bl0EjkFtIJwwgp3sCwiCIAiCIAiCOJohcyIiDs0XsLWQYhWxdm8VPE5FYNEi2u75dB28wTDmn9FHibakpCWHjI+vHYpNxTXNsq4Uq6gnZYtFYJWIPpvIocofVr0w4yP8NLQoMZclfl1NpSnDwY92kmVmBxShMlZoSyRSipziO5sIk8DqftC86gELAMGIpAt6PKeI7Zpopwm3sfYFmmDLs6waxRqB3cyj0hfSE2ZpZTlGsb54efqAOAuB8T0y0dFj139HYiJV2TqRssDBR0xbRS4uGV4sX904XE9w15D3bCLO6N8G763erf9ukiirntPYc8lzDPyqXUMkQaIvQBFdfTEevtv31xrWmyxKU+BY1AYiCeeTfQFBEARBEARBEEczJMoScSiRUK1HlLWIHGoCIZh5Dj/dNkqfXuULYUeZ9zDW7NjFaRYalQ28MTx6Zp+EopLAsQhGJNhMivjWkPCkeZw2R7Tp4bAvOBLh6oiTiTxWeTZ5pKyZ5+APK4Kt4imr2BeEwjERqmp0pmYVoAm8sfYFmh81xyiesopYz6MmEIbAsZAR9RIe2z0LoRjRN5YXLhiASm9I/x0bjRurjXJqXeomw2osY7tnwaNG9ieic6bi/Xmw9gWzJ3UzRvk2oZoMoyZLixHSLQKHGn9YtYmQDcdO+6dJjZQVOBb3TumJuR8bLWQS2RMAiuCb1L6AImUJgiAIgiAIgjiKIVGWiIOr45V4tCOqkVYCx6JNijU6nWexp9yHXWXeViVCH2skS8ylDYtPtYnYUFSti3f1JUZ6afqAZqnT4bAvOBKpGyGbUDznFf/URI4pZoFFIKREjAoci1o1ejQkRQVBjlE6kTQBVEsapiWA++WusdhT7gOg2BfwaqeTReTgU9ctybJ+HZ09MK9eYT72epMR9SCOva5MemKzg7sOslxmZLmyGix3sIm+3FYRT5zVV//dlMhukWeU6ORw9ByYBQ41gbDexjhGsVZgmOgxMPEsvKEIBI6BTeTinjGuBNHugCJsy3Lidk6RsgRBEARBEARBHM2QKEvEwbJRj87WgElg1ezd0Q/4Mm8QAsfiyhEFGDZ/8WGsHdFSiByDcERC50wH/rdmD8wCh3CdodUtBUXKKsRFyiawjrCbOOyt8MdZBQBKdGVAFf94VvGUtYocgjGCoC621znm4YgiDqbaRBRX+dV1sODUIfZmgYNfFWUjktxoITVWHDyzfxuM6eYx7CMA2MRD92htjkutKetwWQR8ft0wZLss+jRNlNU8ZWMjbzXRWOBY+IJh8BwLqxgf4ZrpSBwZrF0zYoJwXkpMRRAEQRAEQRDE0UyLhpmUlZXhvPPOg9PphNvtxiWXXIKamvp9JP1+P6655hqkpaXBbrfj9NNPR3FxsaHMzp07MWnSJFitVng8HvzjH/9AOBzW53/wwQcYO3YsMjIy4HQ6UVhYiC+//LJF9rE1onhvSgflVXgkYuI51KrDlDV4lsHCtUXo6HEcxpoRLQnPsQhGZGS7zNhT4QPHMHj5wgGYNjDvEGy7dbSdv0tdUTZRVGeazYSS6kDCeWae1YVTxVM2DKvIIxQzdJ7n4u1WWAYIhKP3ME2Ij0bKSrAI0WRakiw3WkiNFV/NAqeLk7HCps106MTCQ3mf7pHjRIbDjIIMOywxwqpJYFGrRcqq9gVatbTjwnMMqnxhWEVOT7wWS9s0Kzbce1LcdO2+bUoQFavZUhAEQRAEQRAEQRyNtKgoe95552Ht2rX4+uuv8emnn+KHH37A5ZdfXu8yN954Iz755BO8++67+P7777F3715MnTpVnx+JRDBp0iQEg0EsWbIEr732Gl599VXcfffdepkffvgBY8eOxeeff47Vq1dj1KhROPnkk7FmzZoW29fWBMcyCIZlQzbxoxkTz6I2GDZESD44tRfeumwwslwmeBwmfHnD8MNYQ6IlEDhlWDyv/p/jGPTNcyOzHq/O5qJ3rgtfXD+sxbdzpMPVEUUTRSmn2UWUVPkTCtmxkbICpySYsps4gwetFikbi93EIxCOxEToKo86kWP18hYx+viTJMVG4e8QK47aTIcyUvbv36drA8mTisXy2XXDEibCM3EsfKFIwkRfTEykbKUvBKvIIcUqJlx/Io9YoZ5I2eZIykcQBEEQBEEQBHG4aLEvx/Xr12PhwoVYuXIlBgxQfBqffvppTJw4EY8++ihycnLilqmsrMS///1vLFiwACeeeCIA4JVXXkG3bt2wbNkyDB48GF999RXWrVuHb775BpmZmejbty/uvfde3HbbbZg7dy5EUcSTTz5pWO8DDzyAjz76CJ988gn69evXUrvcalA8GiW0lgTyJl6J4uJjPuo7ehzo6AGq/SG0SbGgSxZFzLY2BNW+AAAiEg5pJwPPseiW7Txk2ztSYfVIWVb9f+JI2eIqP+zmBGIfH/WU1ZKEWUQe1f5osi0+RpR95aKBuOiVlch2WRAMS3GetgzD6IkMzTFD3yVZxt8NbnbHeKJ6HGY8ckbvv7fCRtIc9gVtUiy4YUyng16e51j4gooIHk2wxkCWo4nQBI6BLxSBVeTRLduBT68d2rh1q+e9rqfsnRO7tZrRHARBEARBEARBHJu0mCi7dOlSuN1uXZAFgDFjxoBlWSxfvhynnXZa3DKrV69GKBTCmDFj9Gldu3ZF27ZtsXTpUgwePBhLly5Fr169kJmZqZcZP348rrrqKqxduzah6CpJEqqrq5Gampq0voFAAIFAQP9dVVWlLyvFJJVpjUiSBFmW9f1kICuCBsO0in0XOEbJss4ibn9sIod7TunRKvaTMMIACEvKdR2RJLBM/Plvbuq2pWMdFopYyqj/Z5n4Y2PiGVT7w3BZhLh5vGqlIkkSOEZZB8fAcG/SRElJkjCiUzoAoG2qBf5QBAyU7bHqspJ6HUQkGcVVfv2aiEgyGAbY+sCERp+7uuVSLDzaplogSRIYAKcfl3toroNmuN66ZzvQPdtx0OvhWMAfioBlZMW+gFEtJCIytAxuWqcIA+VYN7Q9rS1pfWl8nfv3JUPbUTsjiEZAzyWCaD6oPRFE80BtiWjtNOXabjFRtqioCB6Px7gxnkdqaiqKioqSLiOKItxut2F6ZmamvkxRUZFBkNXma/MS8eijj6KmpgbTpk1LWt8HH3wQ8+bNi5teWloKv9+fdLnWgCRJqKyshCzLYFkWNYEIarw+WAQOJSUlh7t6f5twREaVN4iA35dwf9J5oKSkdZ/jY5Gq6mowAEpKShAIhpOe/+akbls61vGr1gNV5WUAgIqyMnCB+IjYQDCIUJCNOz9+bw1CEdkwPRQOg4WkT6utqQYA/ffssflYW1SL0gPl8HmDKCkpQUVtSC/jq61FOCJhSK6At6f3UK+PIIKB+O0nY9kN/ROW/b/p3Q/5PbO6uvqw36f9Xi9q/EFUlJUhIsnw1tYgFAzAH5RQUVEOAPDVOU8NobUlVo2KrjiwnyJjCeIgoOcSQTQf1J4IonmgtkS0dqqrqxtdtsmi7O23346HH3643jLr169v6mpbjAULFmDevHn46KOP4kTiWGbNmoWbbrpJ/11VVYW8vDw9WVhrRpKUhDgZGRlgWRb2YBgsvxs2q1jvMTtakGUZwYgMl9PeKvaHaByCuRoWkYPH4wHDrofDbmvx81+3LR3raAm5sjIzAACZngyk2uL9RBluM+xWS9z5SXEHIQOG6YLAg2FlfVrKPiXJo/b7Yo8Hcz5eC7PNDicTgsfjgeAN6mXcLh8iMtChbdRCh+e3wGqJ3/7RgNvlPOz1TnF5EZbLkOlJR0SS4XI6YTGHEEQIGWnKCJXM9BQAaHRdtbZkcynL1e2MJQiicdBziSCaD2pPBNE8UFsiWjtmc+Pz2DRZlL355psxY8aMessUFBQgKysrLiImHA6jrKwMWVlZCZfLyspCMBhERUWFIVq2uLhYXyYrKwsrVqwwLFdcXKzPi+Xtt9/GpZdeinfffddgiZAIk8kEk8kUN51l2WPiRsEwjL6vPMchFJHBMq1n30MRGSLHtZr9IRrmrIFtMaqrByzLIiLL4LlDcz3HtqVjHQFKZKNJUB41Ap+4DUakxOdH8xGNnW4VeZTXBvVpguoNG1tG4DgEI9F1attnWRYCxyEiGXvlQ5J0yK6P5oY7Auot8hz8oQhEnockKx6zDMuAAQNO9R8wi9Fz0FgYhoHIN305giCM0HOJIJoPak8E0TxQWyJaM025rpssymZkZCAjI6PBcoWFhaioqMDq1avRv39/AMCiRYsgSRIGDRqUcJn+/ftDEAR8++23OP300wEAGzduxM6dO1FYWKiv9/7770dJSYkecfP111/D6XSie/fu+rreeustXHzxxXj77bcxadKkpu7mMQ2n+jgmSHZ91KKILjT09Vgiw2FChkPpaIlE5EOa6ItQ0BJ9iVzyRF8AIMlAotNT4w/HLWMTOZRURT16Eq1T4BgEQtE2bxOjSb1iE4NpMDh6r40joe4Cx8AXjOgJ1WLPCaueWBN/cA8Uge7bBEEQBEEQBEG0UlrMU7Zbt2446aSTcNlll+H5559HKBTCzJkzcfbZZyMnRxk2umfPHowePRqvv/46jj/+eLhcLlxyySW46aabkJqaCqfTiWuvvRaFhYUYPHgwAGDcuHHo3r07LrjgAsyfPx9FRUWYPXs2rrnmGj3SdcGCBbjwwgvxz3/+E4MGDdK9Zi0WC1wuV0vtcquBYxgEI3JSAeVoJBSRIbQmlZloEsGIFJe9nTh08ByD7Q8l7xwzCywCoXgz9F5t3LhjYjfDtOtGdzJYIPAJ7lMcy6jnXHnEMQwDl0XQ59WFZRnIshw3nWgcPMvCH5bAqQJqbAeIdrgP9v5LPrIEQRAEQRAEQbRWWlSlePPNN9G1a1eMHj0aEydOxNChQ/Hiiy/q80OhEDZu3Aiv16tPe+KJJzB58mScfvrpGD58OLKysvDBBx/o8zmOw6effgqO41BYWIjzzz8f06dPxz333KOXefHFFxEOh3HNNdcgOztb/7v++utbcndbDSzLIKL6vLQWIpJMEVfHOHT2Dx+JhNNYzDynJwWLpW+eG5cMbW+Y5nGakZ9m038nEll5ThF5Y+f9NmecOi++fJ82LhRk2OKmHw3IOPxiMs8xCIaleiNlfaHIQa//0jrXAEEQBEEQBEEQRGugxSJlASA1NRULFixIOr9du3Zx0UlmsxnPPPMMnnnmmaTL5efn4/PPP086/7vvvmtyXQkjEQmtbri3iecaLkS0UpgjQLo6dmmog8cksAg0QrQrLEiDVTC2Yz6BX4/AMvCGIgkF20TT7jm1Z4PbPlI5EgJ8hTr2FGwCUbZfnhs/3jrqoNY/e3L3hgsRBEEQBEEQBEEcZbSoKEscvUhS67IvAA7e05A4+mGYI0O8IhJj4ln4E9gX1OWtywfHTUt0m+I4Br7aSMI239o6m6Qj4MLWRFlNIDfaFyj/5jkWeanWQ185giAIgiAIgiCIIxRSqYiESLKsf0y3FkwCXe7HKgyODPHqWOTDa4Y0WEbxfD64+42QQHgVWBbeYDihj2lr62w6EtAsIbRDy7EMtNB0SqhLEARBEARBEASRGPpcIhISkeWEEWhHM2ayLzhmsZt51ATCh7saxyR989wNlpFkGVbx4AZu5KdaIdYRXzmWgS8kxU0HEnvKHs0cCZ0Ngqq8ajYVmvDNMGh1nXsEQRAEQRAEQRDNBdkXEAlplfYFFCl7zDKlby4yHKbDXQ0iCU9M62vwIW0KHqcZf90/wTCN5xj4guGEUbRcKwrdvPbEjmiXdvgTlNUVumO1cBJlCYIgCIIgCIIgEkOiLJGQiCwftEhyJNK7jQtmgSJlj1WuG93pcFeBqIcUm9is61MiZSOJI2Vb0X3t5nFdDncVACDOeoJlGMiqf0ErOiktM+QAAQAASURBVNwEQRAEQRAEQRDNComyREIkqXUlxPl45tDDXQWCIA4RPMvAG4xA5OPvYa1tBMCRQF3v3thj3Jo69wiCIAiCIAiCIJqT1jOOk2hWpFboKUsQxLEBx7LwBSMQufjo+CPAgrXVwdexhOAYBizDICLJoMcIQRAEQRAEQRBEYkiUJRISkVqXfQFBEMcOvGpfICSIlA1FpMNQo9ZNnH0By4BjGUiyZmJAEARBEARBEARB1IVEWSIhkty67AsIgjh24FhGjZSNf8QFwyTKNjcm3hiRLPIsWIaBJClJIwmCIAiCIAiCIIh4SJQlEhKRJIqUJQjiqIRXRdm6XqcAEKRI2WYn2202/BY5FrIsAwzgtAi4YQwl2iMIgiAIgiAIgqgLibJEQsKSHDcklSAI4miAYxl4QxGY+PhHXL+2bsw/vfdhqFXrReBYbHtwov5b5FlEZEUcNwscbhjT+TDWjiAIgiAIgiAI4siERFkiIRFJBsfS5UEQxNEHzylJphJFyma7LJg2MO8w1Kp1w8TY3fAsg4gkgaPRFgRBEARBEARBEEkh1Y1ISFiSwdMHNUEQRyFah5KYIFKWaHkEjlWSRZIvOUEQBEEQBEEQRFLoi5VISDhCUU4EQRydaLYFiSJliZbHLHCISKBnCEEQBEEQBEEQRD3wh7sCxJGJJIM8ZQmCOCpxWwUAdA87HPw+dxycZgFnDcxTkn0RBEEQBEEQBEEQCSFRlkgKecoSBHE0kmoVARh9TolDg9OsCOJju2ce5poQBEEQBEEQBEEc2ZDqRiSFPGUJgjgayXCY8L+rTzjc1SAIgiAIgiAIgiCIpJAoSySFp6G/BEEchTAMg35tUw53NQiCIAiCIAiCIAgiKSTKEkmhSFmCIAiCIAiCIAiCIAiCaH5IlCXqgURZgiAIgiAIgiAIgiAIgmhuSJQlCIIgCIIgCIIgCIIgCII4hLSoKFtWVobzzjsPTqcTbrcbl1xyCWpqaupdxu/345prrkFaWhrsdjtOP/10FBcXG8rs3LkTkyZNgtVqhcfjwT/+8Q+Ew+GE6/v555/B8zz69u3bXLt1zGARucNdBYIgCIIgCIIgCIIgCIJodbSoKHveeedh7dq1+Prrr/Hpp5/ihx9+wOWXX17vMjfeeCM++eQTvPvuu/j++++xd+9eTJ06VZ8fiUQwadIkBINBLFmyBK+99hpeffVV3H333XHrqqiowPTp0zF69Ohm37djAYtAoixBEARBEARBEARBEARBNDctJsquX78eCxcuxMsvv4xBgwZh6NChePrpp/H2229j7969CZeprKzEv//9bzz++OM48cQT0b9/f7zyyitYsmQJli1bBgD46quvsG7dOrzxxhvo27cvJkyYgHvvvRfPPPMMgsGgYX1XXnklzj33XBQWFrbUbrZa2qZakek0He5qEARBEARBEARBEARBEESrg2+pFS9duhRutxsDBgzQp40ZMwYsy2L58uU47bTT4pZZvXo1QqEQxowZo0/r2rUr2rZti6VLl2Lw4MFYunQpevXqhczMTL3M+PHjcdVVV2Ht2rXo168fAOCVV17B1q1b8cYbb+C+++5rsL6BQACBQED/XVVVBQCQJAmSJDX9ABxFSJIEWZYN+/ndLSP0eQRBNI5EbYkgiKZDbYkgmgdqSwTRfFB7IojmgdoS0dppyrXdYqJsUVERPB6PcWM8j9TUVBQVFSVdRhRFuN1uw/TMzEx9maKiIoMgq83X5gHApk2bcPvtt+PHH38EzzduFx988EHMmzcvbnppaSn8fn+j1nG0IkkSKisrIcsyWJZyvxHEwUJtiSCaB2pLBNE8UFsiiOaD2hNBNA/UlojWTnV1daPLNlmUvf322/Hwww/XW2b9+vVNXW2zEYlEcO6552LevHno3Llzo5ebNWsWbrrpJv13VVUV8vLykJGRAafT2RJVPWKQJAkMwyAjI4NuigTxN6C2RBDNA7UlgmgeqC0RRPNB7YkgmgdqS0Rrx2w2N7psk0XZm2++GTNmzKi3TEFBAbKyslBSUmKYHg6HUVZWhqysrITLZWVlIRgMoqKiwhAtW1xcrC+TlZWFFStWGJYrLi7W51VXV2PVqlVYs2YNZs6cCSAaHs/zPL766iuceOKJcds2mUwwmeI9VFmWPSZuFAzDHDP7ShAtCbUlgmgeqC0RRPNAbYkgmg9qTwTRPFBbIlozTbmumyzKZmRkICMjo8FyhYWFqKiowOrVq9G/f38AwKJFiyBJEgYNGpRwmf79+0MQBHz77bc4/fTTAQAbN27Ezp079WRdhYWFuP/++1FSUqLbI3z99ddwOp3o3r07BEHAH3/8YVjvs88+i0WLFuG9995D+/btm7rLBEEQBEEQBEEQBEEQBEEQzUaLecp269YNJ510Ei677DI8//zzCIVCmDlzJs4++2zk5OQAAPbs2YPRo0fj9ddfx/HHHw+Xy4VLLrkEN910E1JTU+F0OnHttdeisLAQgwcPBgCMGzcO3bt3xwUXXID58+ejqKgIs2fPxjXXXKNHuvbs2dNQF4/HA7PZHDedIAiCIAiCIAiCIAiCIAjiUNNioiwAvPnmm5g5cyZGjx4NlmVx+umn46mnntLnh0IhbNy4EV6vV5/2xBNP6GUDgQDGjx+PZ599Vp/PcRw+/fRTXHXVVSgsLITNZsOFF16Ie+65pyV3hSAIgiAIgiAIgiAIgiAIollgZFmWD3cljkSqqqrgcrlQWVl5TCT60uwgyNOFIA4eaksE0TxQWyKI5oHaEkE0H9SeCKJ5oLZEtHaaoie2aKTs0YymVVdVVR3mmrQ8kiShuroaZrOZbooE8TegtkQQzQO1JYJoHqgtEUTzQe2JIJoHaktEa0fTERsTA0uibBKqq6sBAHl5eYe5JgRBEARBEARBEARBEARBHC1UV1fD5XLVW4bsC5IgSRL27t0Lh8MBhmEOd3ValKqqKuTl5WHXrl2t3qqBIFoSaksE0TxQWyKI5oHaEkE0H9SeCKJ5oLZEtHZkWUZ1dTVycnIajAanSNkksCyLNm3aHO5qHFKcTifdFAmiGaC2RBDNA7UlgmgeqC0RRPNB7YkgmgdqS0RrpqEIWQ0y8CAIgiAIgiAIgiAIgiAIgjiEkChLEARBEARBEARBEARBEARxCCFRloDJZMKcOXNgMpkOd1UI4qiG2hJBNA/UlgiieaC2RBDNB7UngmgeqC0RRBRK9EUQBEEQBEEQBEEQBEEQBHEIoUhZgiAIgiAIgiAIgiAIgiCIQwiJsgRBEARBEARBEARBEARBEIcQEmUJgiAIgiAIgiAIgiAIgiAOISTKEgRBEARBEARBEARBEARBHEJIlCUIgiAIgiAIgiAIgiAIgjiEkChLEARBEARBEARBEARBEARxCCFRliAIgiAIgiAIgiAIgiAI4hBCoixBEARBEARBEARBEARBEMQhhERZgiAIgiAIgiAIgiAIgiCIQwiJsgRBEARBEARBEARBEARBEIcQEmUJgiAIgiAIgiAIgiAIgiAOISTKEgRBEARBEARBEARBEARBHEJIlCUIgiAIgiAIgiAIgiAIgjiEkChLEARBEATRzHz33XdgGAbffffd4a6Kzty5c8EwzOGuRlJaqn7z589H165dIUlSs6+bII5W1q1bB57n8eeffx7uqhAEQRDEMQuJsgRBEARBHBL++OMPnHHGGcjPz4fZbEZubi7Gjh2Lp59+2lDugQcewIcffnh4Kkm0KqqqqvDwww/jtttuA8tGX3sZhsHMmTObfXterxfPPPMMxo0bh+zsbDgcDvTr1w/PPfccIpFIvcu++eabYBgGdru92evVEK+++ioYhoHZbMaePXvi5o8cORI9e/Zslm3deOONOO6445Camgqr1Ypu3bph7ty5qKmpSVhekiRkZGRg/vz5jVq/1+vF3Llzj6gOEQDYvn07GIbR/1iWRWpqKiZMmIClS5c267b27NmDadOmwe12w+l04tRTT8XWrVsNZbp3745Jkybh7rvvbtZtEwRBEATRePjDXQGCIAiCIFo/S5YswahRo9C2bVtcdtllyMrKwq5du7Bs2TL885//xLXXXquXfeCBB3DGGWdgypQph6/Cf5Phw4fD5/NBFMXDXZVjmv/85z8Ih8M455xzDsn2tm7dimuvvRajR4/GTTfdBKfTiS+//BJXX301li1bhtdeey3hcjU1Nbj11lths9kOST2TEQgE8NBDD8V1lDQnK1euxLBhw3DRRRfBbDZjzZo1eOihh/DNN9/ghx9+MIjnALBixQrs378fkyZNatT6vV4v5s2bB0ARk480zjnnHEycOBGRSAR//fUXnn32WYwaNQorV65Er169/vb6a2pqMGrUKFRWVuKOO+6AIAh44oknMGLECPz6669IS0vTy1555ZWYOHEitmzZgg4dOvztbRMEQRAE0TRIlCUIgiAIosW5//774XK5sHLlSrjdbsO8kpKSg15vbW3tYReyEsGyLMxm8+GuxjHPK6+8glNOOeWQnYusrCz88ccf6NGjhz7tiiuuwMUXX4xXXnkFd911Fzp27Bi33H333QeHw4FRo0b9rSjxdu3aYcaMGZg7d+5BLd+3b1+89NJLmDVrFnJycg66HvXx008/xU3r0KEDbrnlFqxYsQKDBw82zPv888+Rn59vOKZHM8cddxzOP/98/fewYcMwYcIEPPfcc3j22Wf/9vqfffZZbNq0CStWrMDAgQMBABMmTEDPnj3x2GOP4YEHHtDLjhkzBikpKXjttddwzz33/O1tEwRBEATRNMi+gCAIgiCIFmfLli3o0aNHnCALAB6PR/83wzCora3Fa6+9pg/znTFjBoCo5+i6detw7rnnIiUlBUOHDtWXfeONN9C/f39YLBakpqbi7LPPxq5duwzb+vHHH3HmmWeibdu2MJlMyMvLw4033gifz2coN2PGDNjtduzcuROTJ0+G3W5Hbm4unnnmGQCKFcOJJ54Im82G/Px8LFiwwLB8Ik9ZbQj4unXrMGrUKFitVuTm5iYclr1jxw6ccsopsNls8Hg8uPHGG/Hll1822qf2p59+wsCBA2E2m9GhQwe88MILScs25rgBwPLlyzFx4kSkpKTAZrOhd+/e+Oc//2kos2jRIgwbNgw2mw1utxunnnoq1q9ff0jqV5dt27bh999/x5gxYxos21ykp6cnFA9PO+00AEh4LDZt2oQnnngCjz/+OHj+8MZL3HHHHYhEInjooYcO6XbbtWsHAKioqIib99lnnxmiZFetWoXx48cjPT0dFosF7du3x8UXXwxAsQjIyMgAAMybN0+/h8SK1Bs2bMAZZ5yB1NRUmM1mDBgwAB9//LFhm5qdww8//IArrrgCaWlpcDqdmD59OsrLy5t134cNGwZAuUc2B++99x4GDhyoC7IA0LVrV4wePRr/93//ZygrCAJGjhyJjz76qFm2TRAEQRBE06BIWYIgCIIgWpz8/HwsXboUf/75Z73elP/9739x6aWX4vjjj8fll18OAHHDas8880x06tQJDzzwAGRZBqBE4t51112YNm0aLr30UpSWluLpp5/G8OHDsWbNGl0Mfvfdd+H1enHVVVchLS0NK1aswNNPP43du3fj3XffNWwnEolgwoQJGD58OObPn48333wTM2fOhM1mw5133onzzjsPU6dOxfPPP4/p06ejsLAQ7du3r/c4lJeX46STTsLUqVMxbdo0vPfee7jtttvQq1cvTJgwAYAS/XviiSdi3759uP7665GVlYUFCxZg8eLFjTrWf/zxB8aNG4eMjAzMnTsX4XAYc+bMQWZmZlzZxh63r7/+GpMnT0Z2drZep/Xr1+PTTz/F9ddfDwD45ptvMGHCBBQUFGDu3Lnw+Xx4+umnMWTIEPzyyy+68NYS9UvEkiVLACiRiQdDTU0N/H5/g+UEQYDL5aq3TFFREQBFtK3LDTfcgFGjRmHixIlxotmhpn379pg+fTpeeukl3H777fVGy1ZWViIUCjW4TrPZHOeTGw6HUVFRgWAwiD///BOzZ8+Gw+HA8ccfbyhXVFSENWvW6FGcJSUl+rVz++23w+12Y/v27fjggw8AABkZGXjuuedw1VVX4bTTTsPUqVMBAL179wYArF27FkOGDEFubi5uv/122Gw2/N///R+mTJmC999/XxfPNWbOnAm32425c+di48aNeO6557Bjxw6906U52L59OwAgJSXFMD0QCKC6urpR69CuK0mS8Pvvv+sidSzHH388vvrqK1RXV8PhcOjT+/fvj48++ghVVVVwOp0HuRcEQRAEQRwUMkEQBEEQRAvz1VdfyRzHyRzHyYWFhfKtt94qf/nll3IwGIwra7PZ5AsvvDBu+pw5c2QA8jnnnGOYvn37dpnjOPn+++83TP/jjz9knucN071eb9x6H3zwQZlhGHnHjh36tAsvvFAGID/wwAP6tPLyctliscgMw8hvv/22Pn3Dhg0yAHnOnDn6tMWLF8sA5MWLF+vTRowYIQOQX3/9dX1aIBCQs7Ky5NNPP12f9thjj8kA5A8//FCf5vP55K5du8atMxFTpkyRzWazYX/WrVsncxwnx776Nfa4hcNhuX379nJ+fr5cXl5uKCtJkv7vvn37yh6PRz5w4IA+7bfffpNZlpWnT5/eYvVLxuzZs2UAcnV1ddw8API111xT7/LaNdDQ34gRI+pdTyAQkLt37y63b99eDoVChnmffvqpzPO8vHbtWn2bNput3vXVR35+vuE6bCyvvPKKDEBeuXKlvGXLFpnnefm6667T548YMULu0aOHYRntem7oL1FbXrp0qaFMly5dEl7X//73v2WLxaK32//97396PZNRWloa1x41Ro8eLffq1Uv2+/36NEmS5BNOOEHu1KlT3PHo37+/4R41f/58GYD80UcfJd1+MrZt2yYDkOfNmyeXlpbKRUVF8o8//igPHDhQBiC/++67hvJaHRrzV3ff77nnnrjtP/PMMzIAecOGDYbpCxYskAHIy5cvb/I+EQRBEATx96BIWYIgCIIgWpyxY8di6dKlePDBB/Hll19i6dKlmD9/PjIyMvDyyy/jlFNOafS6rrzySsPvDz74AJIkYdq0adi/f78+PSsrC506dcLixYtxxx13AAAsFos+v7a2Fj6fDyeccAJkWcaaNWvQtm1bw7ovvfRS/d9utxtdunTB5s2bMW3aNH16ly5d4Ha747KbJ8Jutxv8JEVRxPHHH29YduHChcjNzTUcE7PZjMsuuww333xzveuPRCL48ssvMWXKFMO+dOvWDePHj8fnn3+uT2vscVuzZg22bduGJ554Ii4yVYsW3LdvH3799VfceuutSE1N1ef37t0bY8eO1bfbEvVLxoEDB8DzfFyUZmO59dZbDecqGXUjHOsyc+ZMrFu3Dp999pnBniAYDOLGG2/ElVdeie7duze5fokiKSVJgtfrNRwvIHGEbjIKCgpwwQUX4MUXX8Ttt9+O7OzshOUee+yxRg3lTxRt2717d3z99deora3FkiVL8M0336Cmpiau3Oeff45Ro0bp7Va7/j799FP06dMHgiA0er/KysqwaNEi3HPPPaiurjYcu/Hjx2POnDnYs2cPcnNz9emXX365YRtXXXUV7rjjDnz++edNumfFMmfOHMyZM0f/bbfb8dhjj+GMM84wlBs/fjy+/vrrJq1bs2ExmUxx8zRf5bpWLdr1W/eaIQiCIAii5SFRliAIgiCIQ8LAgQPxwQcfIBgM4rfffsP//vc/PPHEEzjjjDPw66+/NlqYqmsRsGnTJsiyjE6dOiUsHyuq7Ny5E3fffTc+/vjjOEGpsrLS8NtsNuv+lBoulwtt2rSJG7rscrkaJVAlWjYlJQW///67/nvHjh3o0KFDXLlECaLqUlpaCp/Pl/BYdOnSxSB6Nva4aV6X9dlO7NixQ99GXbp164Yvv/wStbW1qK6ubvb6tRTdu3c/KLE0lkceeQQvvfQS7r33XkycONEw74knnsD+/fsxb968g1r3W2+9hYsuuijhNh955BHDNFm1+Wgss2fPxn//+1889NBDcb7BGv3792/SOmNxOp261++pp56KBQsW4NRTT8Uvv/yCPn36AABCoRC+/vprPPjgg/pyI0aMwOmnn4558+bhiSeewMiRIzFlyhSce+65CYXIWDZv3gxZlnHXXXfhrrvuSlimpKTEIMrWvfbsdjuys7N1y4GD4fLLL8eZZ54Jv9+PRYsW4amnnkIkEokrl52dnVQQT4YmXgcCgbh5mhVHbMcUEL02msuOgSAIgiCIxkOiLEEQBEEQhxRRFPVENJ07d8ZFF12Ed9991xA9Vh91RQVJksAwDL744gtwHBdXXouUjEQiGDt2LMrKynDbbbeha9eusNls2LNnD2bMmAFJkgzLJVpXfdMbI3z9nWWbm8Yet8PF361fWloawuFwnIdmY6msrIyLKkyEKIqG6GCNV199FbfddhuuvPJKzJ49O27d9913H66++mpUVVWhqqoKgOJjK8sytm/fDqvVakiCV5dEkZTnn38+xo0bh+nTpzdmF5NSUFCA888/X4+WTURZWRmCwWCD67JYLA167k6dOhUXXHAB3n77bV2U/emnn1BVVWUQsxmGwXvvvYdly5bhk08+wZdffomLL74Yjz32GJYtW1bvNaG171tuuQXjx49PWKYxHR9/l06dOumC9OTJk8FxHG6//XaMGjUKAwYM0Mv5fL64jqJkZGVlAQBSU1NhMpmwb9++uDLatLqRy1pnUlOiqQmCIAiCaB5IlCUIgiAI4rChiRCxIkJTI7Y6dOgAWZbRvn17dO7cOWm5P/74A3/99Rdee+01g2jV1CHCLU1+fj7WrVsHWZYNx2Lz5s0NLpuRkQGLxYJNmzbFzdu4caPhd2OPm5Zo7c8//9TFpER1TrQNQMl2n56eDpvNBrPZ3Oz1S0bXrl0BANu2bdMTPTWF66+/Hq+99lqD5UaMGIHvvvvOMO2jjz7CpZdeiqlTp+KZZ56JW6a8vBw1NTWYP38+5s+fHze/ffv2OPXUU/Hhhx8m3W6iSEqz2YyCgoKk56kpzJ49G2+88QYefvjhhPOnTp2K77//vsH1XHjhhXj11VfrLRMIBCBJkkGE/Oyzz9C9e3c9QVwsgwcPxuDBg3H//fdjwYIFOO+88/D222/j0ksvTXr/KCgoAKBEWDf2+GzatAmjRo3Sf9fU1GDfvn1xUc9/hzvvvBMvvfQSZs+ejYULF+rT33nnnYSR0InQOnVYlkWvXr2watWquDLLly9HQUFBXAfFtm3bwLLsQbUxgiAIgiD+HiTKEgRBEATR4ixevBgjR46ME0y04eqxw95tNhsqKioave6pU6di1qxZmDdvHt544w3DNmRZRllZGdLS0vRoy9ioVFmWkw7PPlxoEZAff/wxTj31VADK0OOXXnqpwWU5jsP48ePx4YcfYufOnbpv6/r16/Hll18ayjb2uB133HFo3749nnzyScyYMcPgK6sJx9nZ2ejbty9ee+01zJo1Sy/z559/4quvvtK9WVuifskoLCwEAKxateqgRNmD9ZT94YcfcPbZZ2P48OF48803wbJs3DIejwf/+9//4qY/9dRTWLp0Kd56660mD11vbjp06IDzzz8fL7zwAvLz8w1+uMDBecpWVFTAZrPFWU+8/PLLAGCIFP38888xefJkQ7ny8nK43W7DtdC3b18A0SH7VqtV31YsHo8HI0eOxAsvvIBrr7027viWlpbG2ZW8+OKLuOiii/T6PvfccwiHw5gwYUKD+91Y3G43rrjiCsyfPx+//vqrvj8H4ykLAGeccQZuv/12rFq1Sj+eGzduxKJFi3DLLbfElV+9ejV69OjRYDQzQRAEQRDND4myBEEQBEG0ONdeey28Xi9OO+00dO3aFcFgEEuWLME777yDdu3aGSLC+vfvj2+++QaPP/44cnJy0L59ewwaNCjpujt06ID77rsPs2bNwvbt2zFlyhQ4HA5s27YN//vf/3D55ZfjlltuQdeuXdGhQwfccsst2LNnD5xOJ95///1GCUuHkiuuuAL/+te/cM455+D6669HdnY23nzzTT1RT0ORxPPmzcPChQsxbNgwXH311QiHw3j66afRo0cPg3dtY48by7J47rnncPLJJ6Nv37646KKLkJ2djQ0bNmDt2rW6mPrII49gwoQJKCwsxCWXXAKfz4enn34aLpcLc+fObbH6JaOgoAA9e/bEN998g4svvjhu/qpVq3DffffFTR85ciSGDh16UJ6yO3bswCmnnAKGYXDGGWfg3XffNczv3bs3evfuDavViilTpsQt/+GHH2LFihUJ5x0O7rzzTvz3v//Fxo0b0aNHD8O8g/GU/e6773DdddfhjDPOQKdOnRAMBvHjjz/igw8+wIABA3QRfNu2bVi/fj2ee+45w/KvvfYann32WZx22mno0KEDqqur8dJLL8HpdOrRqxaLBd27d8c777yDzp07IzU1FT179kTPnj3xzDPPYOjQoejVqxcuu+wyFBQUoLi4GEuXLsXu3bvx22+/GbYXDAYxevRoTJs2DRs3bsSzzz6LoUOHGpJ8vfrqq7jooovwyiuvYMaMGU0+JoASlf3kk0/ioYcewttvvw3g4DxlAeDqq6/GSy+9hEmTJuGWW26BIAh4/PHHkZmZGZcoMBQK4fvvv8fVV199UPUmCIIgCOJvIhMEQRAEQbQwX3zxhXzxxRfLXbt2le12uyyKotyxY0f52muvlYuLiw1lN2zYIA8fPly2WCwyAPnCCy+UZVmW58yZIwOQS0tLE27j/fffl4cOHSrbbDbZZrPJXbt2la+55hp548aNepl169bJY8aMke12u5yeni5fdtll8m+//SYDkF955RW93IUXXijbbLa4bYwYMULu0aNH3PT8/Hx50qRJ+u/FixfLAOTFixc3uOyFF14o5+fnG6Zt3bpVnjRpkmyxWOSMjAz55ptvlt9//30ZgLxs2bKE+x/L999/L/fv318WRVEuKCiQn3/+ef341aUxx02WZfmnn36Sx44dKzscDtlms8m9e/eWn376aUOZb775Rh4yZIhssVhkp9Mpn3zyyfK6desOSf0S8fjjj8t2u132er2G6QCS/t17770NrjcZ2nlP9jdnzpx6l0923TWW/Pz8BreRiFdeeUUGIK9cuTJhnQAkvHabyubNm+Xp06fLBQUFssVikc1ms9yjRw95zpw5ck1NjV7uX//6l+xyueRQKGRY/pdffpHPOeccuW3btrLJZJI9Ho88efJkedWqVYZyS5Ys0a+vusd9y5Yt8vTp0+WsrCxZEAQ5NzdXnjx5svzee+/FHY/vv/9evvzyy+WUlBTZbrfL5513nnzgwAHDtp5++mkZgLxw4cJ6933btm0yAPmRRx5JOH/GjBkyx3Hy5s2b611PY9i1a5d8xhlnyE6nU7bb7fLkyZPlTZs2xZX74osvZAAJ5xEEQRAE0fIwsnwYMksQBEEQBEEQTeLJJ5/EjTfeiN27dxsyxBPJqaysREFBAebPn49LLrnkcFeHaCQTJ06E3W7H//3f/x2W7WvRrytXrjRYKiRi2rRp2L59O1asWHGIatd8TJkyBQzDJLTSIAiCIAii5SH7AoIgCIIgiCMMn88Hi8Wi//b7/XjhhRfQqVMnEmSbgMvlwq233opHHnkEF110UUJ/V+LIY+TIkRg2bNjhrkaDyLKM7777Dm+88cbhrkqTWb9+PT799FP8+uuvh7sqBEEQBHHMQpGyBEEQBEEQRxgTJkxA27Zt0bdvX1RWVuKNN97A2rVr8eabb+Lcc8893NUjiFZNUyJlCYIgCIIgDhaKlCUIgiAIgjjCGD9+PF5++WW8+eabiEQi6N69O95++22cddZZh7tqBEEQBEEQBEE0AxQpSxAEQRAEQRAEQRAEQRAEcQghYy2CIAiCIAiCIAiCIAiCIIhDCImyBEEQBEEQBEEQBEEQBEEQhxDylE2CJEnYu3cvHA4HGIY53NUhCIIgCIIgCIIgCIIgCOIIRpZlVFdXIycnByxbfywsibJJ2Lt3L/Ly8g53NQiCIAiCIAiCIAiCIAiCOIrYtWsX2rRpU28ZEmWT4HA4ACgH0el0HubatCySJKG0tBQZGRkNqvgEQSSH2hJBNA/UlgiieaC2RBDNB7UngmgeqC0RrZ2qqirk5eXpumJ9kCibBM2ywOl0HhOirN/vh9PppJsiQfwNqC0RRPNAbYkgmgdqSwTRfFB7IojmgdoScazQGCtUagEEQRAEQRAEQRAEQRAEQRCHkEMiyj7zzDNo164dzGYzBg0ahBUrVtRb/t1330XXrl1hNpvRq1cvfP7554b5sizj7rvvRnZ2NiwWC8aMGYNNmzYZyvz111849dRTkZ6eDqfTiaFDh2Lx4sXNvm8EQRAEQRAEQRAEQRAEQRBNocVF2XfeeQc33XQT5syZg19++QV9+vTB+PHjUVJSkrD8kiVLcM455+CSSy7BmjVrMGXKFEyZMgV//vmnXmb+/Pl46qmn8Pzzz2P58uWw2WwYP348/H6/Xmby5MkIh8NYtGgRVq9ejT59+mDy5MkoKipq6V0mCIIgCIIgCIIgCIIgCIJICiPLstySGxg0aBAGDhyIf/3rXwAU/5C8vDxce+21uP322+PKn3XWWaitrcWnn36qTxs8eDD69u2L559/HrIsIycnBzfffDNuueUWAEBlZSUyMzPx6quv4uyzz8b+/fuRkZGBH374AcOGDQMAVFdXw+l04uuvv8aYMWMarHdVVRVcLhcqKyuPCU/ZkpISeDwe8nQhiL8BtSWCaB6oLRFE80BtiSCaD2pPBNE8UFsiWjtN0RNbNNFXMBjE6tWrMWvWLH0ay7IYM2YMli5dmnCZpUuX4qabbjJMGz9+PD788EMAwLZt21BUVGQQVl0uFwYNGoSlS5fi7LPPRlpaGrp06YLXX38dxx13HEwmE1544QV4PB70798/4XYDgQACgYD+u6qqCoByw5Ak6aD2/2hBkiTIstzq95MgWhpqSwTRPFBbIojmgdoSQTQf1J4IonmgtkS0dppybbeoKLt//35EIhFkZmYapmdmZmLDhg0JlykqKkpYXrMd0P5fXxmGYfDNN99gypQpcDgcYFkWHo8HCxcuREpKSsLtPvjgg5g3b17c9NLSUoMtQmtEkiRUVlZClmXqqSKIvwG1JYJoHqgtEUTzQG2JIJoPak8E0TxQWyJaO9XV1Y0u26Ki7OFClmVcc8018Hg8+PHHH2GxWPDyyy/j5JNPxsqVK5GdnR23zKxZswwRulVVVcjLy0NGRsYxYV/AMAwyMjIad1MsWQ/mj3chj7675StHEEcRTW5LBEEkhNoSQTQP1JYIovmg9kQQzQO1JaK1YzabG122RUXZ9PR0cByH4uJiw/Ti4mJkZf0/e+cdL0V1/v/PzPa9vV9AEERQsICiEoxdIpYYSYw9FmKw5GeiIWq+NuwxsdcES1RMNPYQKwbBLqKCXUFQ+uX2snf77sz8/njm7JnZnb2Ne4GLz/v1uq+9uzs7c6acM3M+5zmfp9bxN7W1tV0uL14bGhps4mpDQwMmTpwIAFi0aBFeeukltLW1ZQTVv/3tb1iwYAHmzp3r6GXr8/ng8/lyPldV9QfRUCiK0vN9Xf0WsGQOlJ9cM+DlYpjBRq/qEsMweeG6xDD9A9clhuk/uD4xTP/AdYnZnunNdT2gNcDr9WLSpElYuHBh5jNd17Fw4UJMmTLF8TdTpkyxLQ8ACxYsyCw/atQo1NbW2pYJhUJYsmRJZploNAog90Coqsq+Jf2FomztEjAMwzAMwzAMwzAMwzDMoGTA7QtmzZqFM888E/vssw/2228/3HnnnYhEIpgxYwYA4IwzzsCwYcNw0003AQAuvPBCHHzwwbjttttwzDHH4Mknn8THH3+MBx54AACNqFx00UW44YYbMGbMGIwaNQpXXXUVhg4diunTpwMgYbesrAxnnnkmZs+ejUAggAcffBCrV6/GMcccM9C7/APA2NoFYBiGYRiGYRiGYRiGYZhBy4CLsieddBKampowe/Zs1NfXY+LEiZg/f34mUde6detsEa37778/nnjiCVx55ZW4/PLLMWbMGMybNw+77757ZplLL70UkUgE55xzDtrb23HAAQdg/vz5Gd+GyspKzJ8/H1dccQUOO+wwpFIp7Lbbbvjvf/+LCRMmDPQub/+kE4C75x4ZDMMwDMMwDMMwDMMwDMNIFMMwOOzRgVAohJKSEnR0dPwgEn01Njaiurq6Z94Xi24EPn0CmPXVwBeOYQYRva5LDMM4wnWJYfoHrksM039wfWKY/oHrErO90xs9kWsA03vSccCdmxSNYRiGYRiGYRiGYRiGYZjuYVGW6T1aEmj9DljywNYuCcMwDMMwDMMwDMMwDMMMOliUZXpPOkGvy1/auuVgGIZhGIZhGIZhGIZhmEEIi7JM79GS5j9sR8wwDMMwDMMwDMMwDMMwvcW9tQvADELSCWDCKYCube2SMAzDMAzDMAzDMAzDMMyggyNlmd6jJYDDrgI8/q1dEoZhGIZhGIZhGIZhGIYZdLAoy/QeLQX4CoFUfGuXhGEYhmEYhmEYhmEYhmEGHSzKMn3DEwTSLMoyDMMwDMMwDMMwDMMwTG9hUZbpAwqgugE9vbULwjAMwzAMwzAMwzAMwzCDDhZlmb6hKFu7BAzDMAzDMAzDMAzDMAwzKGFRlmEYhmEYhmEYhmEYhmEYZgvCoizDMAzDMAzDMAzDMAzDMMwWhEVZhmEYhmEYhmEYhmEYhmGYLQiLskwfMMwXY+sWg2EYhmEYhmEYhmEYhmEGISzKMn2Ak3wxDMMwDMMwDMMwDMMwTF9hUZbpOwqLswzDMAzDMAzDMAzDMAzTW1iUZRiGYRiGYRiGYRiGYRiG2YKwKMswDMMwDMMwDMMwDMMwDLMFYVGW6R26Bih82TAMwzAMwzAMwzAMwzBMX2F1jekdWgpwebZ2KRiGYRiGYRiGYRiGYRhm0MKiLNM7tCTg8m7tUjAMwzAMwzAMwzAMwzDMoGWLiLL33XcfRo4cCb/fj8mTJ+PDDz/scvlnnnkGu+66K/x+P/bYYw+88sortu8Nw8Ds2bMxZMgQBAIBTJ06FStXrsxZz8svv4zJkycjEAigrKwM06dP78/d+mGipViUZRiGYRiGYRiGYRiGYZjNYMBF2aeeegqzZs3C1VdfjWXLlmHChAmYNm0aGhsbHZd///33ccopp+Dss8/GJ598gunTp2P69On48ssvM8vcfPPNuPvuuzFnzhwsWbIEBQUFmDZtGuLxeGaZ5557DqeffjpmzJiBzz77DO+99x5OPfXUgd7d7Ze3bgESnWakLNsXMAzDMAzDMAzDMAzDMExfUQzDMAZyA5MnT8a+++6Le++9FwCg6zqGDx+O3/3ud/i///u/nOVPOukkRCIRvPTSS5nPfvSjH2HixImYM2cODMPA0KFD8cc//hEXX3wxAKCjowM1NTV49NFHcfLJJyOdTmPkyJG49tprcfbZZ/ep3KFQCCUlJejo6EBxcXGf1jFY0HUdjY2NqK6uhqrm0emvKQEu/Ax4/ARgp0OBo28G/n0KcMq/t2xhGWYbpkd1iWGYbuG6xDD9A9clhuk/uD4xTP/AdYnZ3umNnugeyIIkk0ksXboUl112WeYzVVUxdepULF682PE3ixcvxqxZs2yfTZs2DfPmzQMArF69GvX19Zg6dWrm+5KSEkyePBmLFy/GySefjGXLlmHjxo1QVRV77bUX6uvrMXHiRNxyyy3YfffdHbebSCSQSCQy70OhEABqMHRd79P+DxZ0XYdhGF3upwpAT6egNn8LY+cjYOg6FMOAsZ0fG4bpDT2pSwzDdA/XJYbpH7guMUz/wfWJYfoHrkvM9k5vru0BFWWbm5uhaRpqampsn9fU1GD58uWOv6mvr3dcvr6+PvO9+CzfMt9//z0A4JprrsHtt9+OkSNH4rbbbsMhhxyCb7/9FuXl5Tnbvemmm3DttdfmfN7U1GSzRdge0XUdHR0dMAwj70hVLYCW5kZUAYhGQuhsbERpMoH2hgZAUbZoeRlmW6UndYlhmO7husQw/QPXJYbpP7g+MUz/wHWJ2d7p7Ozs8bIDKspuLYQqfcUVV+D4448HADzyyCPYYYcd8Mwzz+Dcc8/N+c1ll11mi9ANhUIYPnw4qqqqfhD2BYqioKqqqstGsaKUjkPQ60KguhqKvwDVVRWAul1eRgzTa3palxiG6RquSwzTP3BdYpj+g+sTw/QPXJeY7R2/39/jZQdUTausrITL5UJDQ4Pt84aGBtTW1jr+pra2tsvlxWtDQwOGDBliW2bixIkAkPl8/Pjxme99Ph922mknrFu3znG7Pp8PPp8v53NVVX8QDYWiKN3uqwqyH1b0FBRVBVxuKIYGqN4tVUyG2ebpSV1iGKZ7uC4xTP/AdYlh+g+uTwzTP3BdYrZnenNdD2gN8Hq9mDRpEhYuXJj5TNd1LFy4EFOmTHH8zZQpU2zLA8CCBQsyy48aNQq1tbW2ZUKhEJYsWZJZZtKkSfD5fFixYkVmmVQqhTVr1mDHHXfst/37waGn6TWdpFfVIz9jGIZhGIZhGIZhGIZhGKZHDPi881mzZuHMM8/EPvvsg/322w933nknIpEIZsyYAQA444wzMGzYMNx0000AgAsvvBAHH3wwbrvtNhxzzDF48skn8fHHH+OBBx4AQCMqF110EW644QaMGTMGo0aNwlVXXYWhQ4di+vTpAIDi4mKcd955uPrqqzF8+HDsuOOOuOWWWwAAJ5xwwkDv8vaLrtFrtJleVTegpbZeeRiGYRiGYRiGYRiGYRhmEDLgouxJJ52EpqYmzJ49G/X19Zg4cSLmz5+fSdS1bt06W2jv/vvvjyeeeAJXXnklLr/8cowZMwbz5s3D7rvvnlnm0ksvRSQSwTnnnIP29nYccMABmD9/vs234ZZbboHb7cbpp5+OWCyGyZMnY9GiRSgrKxvoXd5+0dPAkAnAkX+l9y63FGoZhmEYhmEYhmEYhmEYhukRimEYxtYuxLZIKBRCSUkJOjo6fhCJvhobG1FdXZ3f++KaEuCM/wIfPgic/Dh99tIfgIMuBYqHOP+GYX5g9KguMQzTLVyXGKZ/4LrEMP0H1yeG6R+4LjHbO73RE7kGMD0nFQdcHvledQM62xcwDMMwDMMwDMMwDMMwTG9gUZbpOekYJfcScKIvhmEYhmEYhmEYhmEYhuk1LMoyPScnUtYFaCzKMgzDMAzDMAzDMAzDMExvYFGW6TnpGFkWCFQ3R8oyDMMwDMMwDMMwDMMwTC9hUZbpOak44PLK9y62L2AYhmEYhmEYhmEYhmGY3sKiLNNz0jFO9MUwDMMwDMMwDMMwDMMwmwmLskzPSScc7Au0rVcehmEYhmEYhmEYhmEYhhmEsCjL9JyUU6Qs2xcwDMMwDMMwDMMwDMMwTG9gUZbpOamY3VNWdQMa2xcwDMMwDMMwDMMwDMMwTG9gUZbpHl2n13TMbl/Aib4YhmEYhmEYhmEYhmEYptewKMt0j6EBLh95ytrsC1wsyjIMwzAMwzAMwzAMwzBML2FRlukeXSPbglQMUK2iLEfKMgzDMAzDMAzDMAzDMExvYVGW6R5DA9xeIB3nRF8MwzAMwzAMwzAMwzAMs5mwKMt0j27aF6QcRFlO9MUwDLP909kAGMbWLgXDMAzDMAzDMMx2A4uyTPdkImWz7AtcbhJsGYZhmO2b28YCTSu2dikYhmEYhmEYhmG2G1iUZbqnq0hZnSNlGYZhfhAoytYuAcMwDMMwDMMwzHYDi7JM9+iWSFn2lGUYhvlhorq3dgkYhmEYhmEYhmG2G1iUZbrHsETKWu0LVI9dlN302ZYvG8MwDMMwDMMwDMMwDMMMMliUZbpH1wC3zyFS1gVoFlH2/oPs7xmGYZjtB/YQZxiGYRiGYRiG6TdYlGW6R08DLq8ZKWuZvury5NoXhDZu2bIxDMMwWwa2q2EYhmEYhmEYhuk3togoe99992HkyJHw+/2YPHkyPvzwwy6Xf+aZZ7DrrrvC7/djjz32wCuvvGL73jAMzJ49G0OGDEEgEMDUqVOxcuVKx3UlEglMnDgRiqLg008/7a9d+mFh6CTKpmP0KnDylI22bNmyMQzDMFsGgyNlGYZhGIZhGIZh+osBF2WfeuopzJo1C1dffTWWLVuGCRMmYNq0aWhsbHRc/v3338cpp5yCs88+G5988gmmT5+O6dOn48svv8wsc/PNN+Puu+/GnDlzsGTJEhQUFGDatGmIx+M567v00ksxdOjQAdu/HwTCvgDIEmU9gJayLxtt3XLlYhiGYbYcHCnLMAzDMAzDMAzTbwy4KHv77bdj5syZmDFjBsaPH485c+YgGAzi4Ycfdlz+rrvuwpFHHolLLrkE48aNw/XXX4+9994b9957LwCKkr3zzjtx5ZVX4rjjjsOee+6Jxx57DHV1dZg3b55tXa+++ir+97//4dZbbx3o3dy+MayirMVT1u0D0llCeLJzy5WLYRiG2XKwpyzDMAzDMAzDMEy/MaCibDKZxNKlSzF16lS5QVXF1KlTsXjxYsffLF682LY8AEybNi2z/OrVq1FfX29bpqSkBJMnT7ats6GhATNnzsQ///lPBIPB/tytHx66JiNkrZGybn+uKJvKjVbO0Fnf/2VjGIZhtgwcKcswDMMwDMMwDNNvuLtfpO80NzdD0zTU1NTYPq+pqcHy5csdf1NfX++4fH19feZ78Vm+ZQzDwFlnnYXzzjsP++yzD9asWdNtWROJBBKJROZ9KBQCAOi6Dl3Xu/39YEbXdRiGkX8/tRQUlwcKAF1xAWI5lxdKKgZD1wHDgAIFRioqv89CvW0X6LPbBmYnGGYboNu6xDCDFBWAnk7lbd/7G65LDNM/cF1imP6D6xPD9A9cl5jtnd5c2wMqym4t7rnnHnR2duKyyy7r8W9uuukmXHvttTmfNzU1OXrVbk/ouo6Ojg4YhgFVzQ2edrc0IZDUUQCgpb0TmkF+wEosjKJwO0KNjYCWRLW3COG2JkTz+AXXAmhsaAAUZQD3hmG2Ht3VJYYZrNQCaG9tRjLo3L73N1yXGKZ/4LrEMP0H1yeG6R+4LjHbO52dPbf1HFBRtrKyEi6XCw0NDbbPGxoaUFtb6/ib2traLpcXrw0NDRgyZIhtmYkTJwIAFi1ahMWLF8Pn89nWs88+++C0007D3Llzc7Z72WWXYdasWZn3oVAIw4cPR1VVFYqLi3u4x4MTXdehKAqqqqqcG8VkCZSiUgBARXUtUFJtfh6E4gb81dVAvANKsAyFfjcKq6tz12EYAIDqyjK7BQLDbEd0W5cYZhBTWlwIOLXvAwDXJYbpH7guMUz/wfWJYfoHrkvM9o7f7+/xsgMqynq9XkyaNAkLFy7E9OnTAVAFXLhwIS644ALH30yZMgULFy7ERRddlPlswYIFmDJlCgBg1KhRqK2txcKFCzMibCgUwpIlS3D++ecDAO6++27ccMMNmd/X1dVh2rRpeOqppzB58mTH7fp8vhwRFyAP3B9CQ6EoShf7amQSfakePyCW8QSBdAKKqgJ6CgiUQYm2ALFWoKDSvopUjH6/6TNgzdvAQZcM4N4wzNaj67rEMIMXVYFs/7cAXJcYpn/gusQw/QfXJ4bpH7guMdszvbmuB9y+YNasWTjzzDOxzz77YL/99sOdd96JSCSCGTNmAADOOOMMDBs2DDfddBMA4MILL8TBBx+M2267DccccwyefPJJfPzxx3jggQcAUOW96KKLcMMNN2DMmDEYNWoUrrrqKgwdOjQj/I4YMcJWhsLCQgDA6NGjscMOOwz0Lm9/6GnAZQrWquWScbllNu5UDAiUAR/8DQhtBE58zL6OZIRev1sIvHsHi7IMwzCDDYN9vxiGYRiGYRiGYfqLARdlTzrpJDQ1NWH27Nmor6/HxIkTMX/+/EyirnXr1tlU5P333x9PPPEErrzySlx++eUYM2YM5s2bh9133z2zzKWXXopIJIJzzjkH7e3tOOCAAzB//vxehQgzvUDXALdpOZDPeiAdBwKl9H8ymvu9EGV1DVB4NIxhGGbQwaIswzAMwzAMwzBMv7FFEn1dcMEFee0K3nzzzZzPTjjhBJxwwgl516coCq677jpcd911Pdr+yJEjYZiepkwfMDQZKdulKFtG/6sOl1XKFGrjHYC3oOvtJcJAy0pg6F59Ky/DMAzT/7AoyzAMwzAMwzAM029wyCLTPboOqC763+XJ/T4VB755ySLKunKXEZGy8XbA040o+8LvgAcO6WtpGYZhmIGARVmGYRiGYRiGYZh+g0VZpnsMTQqtipL7fWgj8PbNgL+U3jtFyiYjgOoxI2WDXW/vq+c3q7gMwzDMAMCiLMMwDMMwDMMwTL+xRewLmEGOngYUh+jXzPdmsi9hS+DkGZuKAf5iINYOeAJdb2/sUQDYboJhtiqd9cCGj4Bxx27tkjDbCizKMgzDMAzDMAzD9BscKct0j645R78KtIRcDiARN2eZJIm28Y7u7QtYkGWYgSWd6H6Z798EnvrVgBeFGUSwNzvDMAzDMAzDMEy/waIs0z1W+wIn0kl6dXmAslFSnLWipwBPsGf2BQAAB5sEhmH6hxtqgI1Lu16mq+h45ocJR8oyDMMwDMMwDMP0GyzKMt2ja10LNFoC2PknwG7TgQs/hWOkq5Ym24J0vAdij0Lr4KgshhkgDCAZ7XoRJ/9oZtvjk8cBLWX/THOYrdAfcJvMMAzDMAzDMAzTb7Aoy3SPrgGqClzTkfud6qIkXjtPBQJlzr9f/jLQ/C3ZFjhZGzjh8uQKDQzD9CPdCGxO3tDMtsd/f0v+v1bmnTcw7SdHyjIMwzAMwzAMw/Qb3OtmusfowlPWEyBLArfXsnyW2PPkqcDa98i2QEv2bJueIJCO9a28DMN0T3dRjz3xnWW2DZLhrPcRZxuZzYVFWYZhGIZhGIZhmH6DRVmme7qyL3D7SJR1+bpeRypGQmtPo7c8AfoNwzADRDeibCrKvrKDhWTE/l7XaDCtv2FRlmEYhmEYhmEYpt9gUZYBAHjql+W3Fugq0ZdbRMpaRNlsL0pPARBpIlEWBk2LzhfFJaL33AEShRiGGRi6i5QVAynMtk/2AJahcaQswzAMwzAMwzDMNg6LsgwAoGLeKUDzSucvexIp6+4iUtbjBxKdFP0KkNVBvqnRWpL8ZDlSlmEGlu4iKVMxWWeZbZvsc6mnOVKWYRiGYRiGYRhmG4dFWUaSTwTVu4iUFZ6yXdkXuAMkytaMB/Y+k5bV8oiy6QQJvJ4gkIr3rvwM80Pl7VuB79/s3W+0bpLupaIsyg4Wss+lrgG6DnQ20Gt/waIswzAMwzAMwzBMv8GiLAO0raXXfKJsl/YFvtxEX07LwAB22Bf42d1mpGwSWPU68NU8+7JakkRbj5/tCximpyybC6x+O/fzaCsJc050l3SPI2UHD9nWM4ZO7fZtY4ENH27++g2DZkuwKMswDMMwDMMwDNNvsCjLAN++Sq/ZGbwFXdkXeII9iJT106vqoVcRKfvpE8BHD9mXTSfYvoBheosrjyXIa5cDT5/h/JvuBDY9DajuzS8bM/DoWQkUdYunbH8ka9M1apdZlGUYhmEYhmEYhuk3WJRlgEApvSZCzt93FSnrLQBirT2IlAV16gFTQEoCWipX9NGS0r4gzaIs8wMnFQdCm7pfzuWl+pSN6s4V7ATdeY4aGiXlY7Z9ss+91VNWtLubg6HRoFp3yeEYhmEYhmEYhmGYHsM9bgbY7Rf0Gu9w/l5P54+28hbSFGkRDetEJlLWFGDdXoqU1bVcUTYdp+Xdfo6UZZhP/wX8bXL3y7k8znYELo+zWAt0L7B15SXNAM2rgEU3bO1SEDn2BVr/RrXqGuByc6QswzAMwzAMwzBMP8KiLENRdgCQjDh/r+v5pzF7C0iU7dK+IDtS1kcCkp7KjcRLRWmdniB7yjKMrvUs4Z3qzhXmgPwRtGLdXWF0YVvCAI1fA2/fsrVLQeREylrsC/Kd/95giHsAR8oyDMMwDMMwDMP0FyzKMhInUQcwp67muVS8BUCys4f2BeYyItGXngZWviaXe/SnQKKT/GQ9gZ6JUQyzXaP0YlGHZfOJtUBu1GMyaxBE1zlStiu2pSRo2RYVhi7Pbz77it5gmLMaOFKWYRiGYRiGYRim32BRlpHki5zrKtGXt4BerZGy1mnRhoGMsCSibUWiL3+puX6zo7/mHYq6zYiyHCnL/MBJxwBPF9Yggr54fVo9ZcNNwJ+H5H7PkbL5cXcxO8CJUN3AlEN1O3vK9mekrC48ZXspysZDwNr3N3/7DMMwDMMwDMMw2yEsyjIAgJbp/4bSVUKgvIm+Cuk1X6SsYYm2E/YFbh9FyqbjwPjpdvE11krWBW4/fc8wP2Ri7UCgbPPW4RRBC9gHYdrWOH/PkbL56cqyJZtoK3D7uIEph+rJjYbWNSm690ukrN43T9mm5cAjR23+9hmGYRiGYRiGYbZDWJRlAAC6tzj/NGc93bWnLGBP9GUVgfS0FHZU4SlrJvoydMBXZE/oleik7z0B+jyd6NsOMcz2QP3nQGFNDxfuhdUBFLvAFm3JXcTQcj2fV78DzPttL7azPeMQnbzkAeCVS3I/T4QGrhhOydwMq6dsnna9N/Q1UpZhGIZhGIZhGIbJyxYRZe+77z6MHDkSfr8fkydPxocfftjl8s888wx23XVX+P1+7LHHHnjllVds3xuGgdmzZ2PIkCEIBAKYOnUqVq5cmfl+zZo1OPvsszFq1CgEAgGMHj0aV199NZJJh+zkDKG6uhBle2lfYPutKehO/JXFU9Zniq2KmdArIqdfJ8K0vNsPLL4XeOF3fd4lhhn0uLxAoLzrZTobkBEIv38TiLV1v15FtdsXxFpzl3GKlG1aDnz5XO6ydZ8Cy1/ufruDmefPMY+1iZNA2bLSebp+tl9vf6K6c6Nh+zNS9oXfm5GyfRBlWcRlGIZhGIZhGIbJy4CLsk899RRmzZqFq6++GsuWLcOECRMwbdo0NDY2Oi7//vvv45RTTsHZZ5+NTz75BNOnT8f06dPx5ZdfZpa5+eabcffdd2POnDlYsmQJCgoKMG3aNMTjNN19+fLl0HUd999/P7766ivccccdmDNnDi6//PKB3t3Bi+rO7ylr6PkTfXlMUTb7eyGyClF2+n1yGZcX0JIADMAbJMFCMwXzRCd1/kUSHY6UZZj8GAZw21j6X1GAx44DVi20f++E6rJ/F20BvEVZ69ZzB2MMPTd6FgA+fhh4+9bel38w8flTdqsVx/ZScRYik5EBKxZFymYNqNkSfW1mpOyyuXLGQ29F1nz3FIZhGIZhGIZhGGbgRdnbb78dM2fOxIwZMzB+/HjMmTMHwWAQDz/8sOPyd911F4488khccsklGDduHK6//nrsvffeuPfeewFQlOydd96JK6+8Escddxz23HNPPPbYY6irq8O8efMAAEceeSQeeeQRHHHEEdhpp53ws5/9DBdffDGef/75gd7dQYuhuPJHVHUVKevkJWuNutW1XOsDESlrGCTqpmLSwiBpiZRlGKZrRD2zCqxWL2Y9JSPUrSiqXTCLtgLBLO9ap0jZfKJstAUIVvSu7IMRqyhpOAiOiuoshA9k0kIRKatrlLANMBN9CVFWA1q+A/57Qd+3oSVN+4JeJpRzOkYMwzAMwzAMwzAMgAEWZZPJJJYuXYqpU6fKDaoqpk6disWLFzv+ZvHixbblAWDatGmZ5VevXo36+nrbMiUlJZg8eXLedQJAR0cHysu7mQb8Q0Z1981Ttrt1WT1lBS4fRcS6feQt++2rUpRNdNLvRaQswzD50ayWLArVHWt0uZaUCfZ0HWj6loQ1xWUXzKLNQEGVwwayfGoNPfczgIQ/J/G3Lzz6U+Db1/pnXf2NVZR1igJVFDh6zW5utGpXiPZ2+csyatpqX2DowMalwCf/7Ps2UrG+2RdwpCzDMAzDMAzDMExeeqG09Z7m5mZomoaaGnuimpqaGixfvtzxN/X19Y7L19fXZ74Xn+VbJptVq1bhnnvuwa235p9em0gkkEhIMSMUosQsuq5D17dvXzxd12EoLhhaGobDvip6GgYUGXmV/X2w0vY7RXHB0FIk/qST9N76W9UDpXMTZZVftQhK3VLoe54MxeUFkhEYihuAAhUkbziViWG2RXRdh2EY/dNmGAYUUwTL1IFUlARVt+nhnIqb9cQAYABuP4xUPFNXlXQSUN0wkjEgVAf1vn2hX9EIxe2FoWtyuUgzECiz12NTXLTVP12DouTWSQUGYOj9UleVTZ/CaFoB7PyTzV5Xf6IC0DVLBKqWps+sx8wAYBj247D2PUBL5SzbXyguD5BOwUiGoRp0v1IMDYbYppYGDK3P21cB6KkYFNPixvj6BWDE/kCwB4OcDseop/RrXWKYHzBclxim/+D6xDD9A9clZnunN9f2gIqy2wIbN27EkUceiRNOOAEzZ87Mu9xNN92Ea6+9NufzpqamjFft9oqu6+jsDKMo0onUW/chtuvxtinKxdEIwq1t0GN5AqvPeA+weASXJFOIfT4fZfN/i6ZTXkNBIoVOy/eeUBiB9kYYLh+MIZNRWLcUrQ0bUeYrgRZuRWeoE6nGRtQCiCcS6MjjP8ww2xq6rqOjowOGYUDN58PcE1IxeOuXIZgmsbN900bA5UHZyzORLt8ZnVP+BABQI42oBpBK60hHo/ArbkQ6WhEx60xJJARf80qof65F84kvohJAY30dqhQPoqGOzHKlcYpUb7fUtVJzkMr6WbAzhEJDyfEEL00kAEWxLdtXqlwBRNsaM2XbVqgF0NrShLRBNg++9jaUAbZjURSLwZdOotnyWe3cn6LtyL/Tsg0NZjRt/1GhK0iGO5Bua0GJWZ5qLY221hZUAOjsaAO0JIpVd14v966oBdDWVI/CtI5UuBOFT5+O5l88g3T1nt3+1tvWinKgT9vtt7rEMD9wuC4xTP/B9Ylh+geuS8z2TmdnZ4+XHVBRtrKyEi6XCw0NDbbPGxoaUFtb6/ib2traLpcXrw0NDRgyZIhtmYkTJ9p+V1dXh0MPPRT7778/HnjggS7Letlll2HWrFmZ96FQCMOHD0dVVRWKi4u73tFBjq7rUFMR+L0eBN+6EkXjpwJVu2S+V3xe+KtqehYZBUAJFMDnSUPRU6gsK4VSUIRAdbVcIF0LZVUKKK6FcfCfYMTrUV4UgFJQAdWIo6yiCjCX9/v88Fl/uz2y/CWKChTRj9sCiU7AV9T9cowNXdehKAqqqqo27wGjZRWU1S8ARRWAlkR1eRHgK4YSb4TXN07Wp3YaMPL4/PD4PICqoqCwAAXm94pHBTTyii4PUnNfXVEKxRdEQUEQBVWVABQoXi+gKKi21DXFR9ej9TMUBKG4XPbPACg+f+6yfURxuVBQUJjZh22J8tKSTNuEtkIA9n1WCgoAh+NTUmwuW1nWfzYPYps+P9x+D4wCPwxFRXV1NRRDR1kJ3beKCguAeBrwFvT5/JQV+qH4g/AGyVamvKRIHoeucDhGPaXf6hLD/MDhusQw/QfXJ4bpH7guMds7fn/PcyQNqCjr9XoxadIkLFy4ENOnTwdAFXDhwoW44ALnpCNTpkzBwoULcdFFF2U+W7BgAaZMmQIAGDVqFGpra7Fw4cKMCBsKhbBkyRKcf/75md9s3LgRhx56KCZNmoRHHnmk28ru8/ng8+WKYqqq/jAaCpcXiulBqCbDgHWfDR2K22P/rCtUF5Q0Rd6p0AGXG4r1tx4/iX6eABRvAZCOQ9HiZGcQ2gjF7c1sS1Fg/+32yNOnAxcsBSp33tolIeIdwF9HANd0bO2SDEoURZHtxp93AP60BnD1sqnVkpQ8q7oGSEWgdNYDcw4kO1dvUNYJMzmfonrIV1RRyPFVfK+lMr6zqm6+Ghrg9kOBDjx6DLD3GWb0pgJFUSyRnIq5bkv909PUVmTXScVh2eaVQNlI6WnbU/Q0FJe75+3NFkRVYCkX2TvY7g+KSu1lVtlV0bbqaWr/1n8IDN+vnwpF557OnUrbNjQ6v1CoDU5FAG9Rn9tSVRfexIbcH0Oj/c32DLdiXkp9vYfa6hLDMH2G6xLD9B9cnximf+C6xGzP9Oa6HvAaMGvWLDz44IOYO3cuvvnmG5x//vmIRCKYMWMGAOCMM87AZZddlln+wgsvxPz583Hbbbdh+fLluOaaa/Dxxx9nRFxFUXDRRRfhhhtuwAsvvIAvvvgCZ5xxBoYOHZoRfjdu3IhDDjkEI0aMwK233oqmpibU19fn9ZxlQB1rkYzGFFQz6GnysezNukSyIackYS4fEA9RZKiiABuX0ftAGZAIy+WPuKHfo8q2WQYyEVBvSSe7X4bJJRnJTaaU7KRkdr0lnQAiTYC3gOpLpBHoWEc+zdb6JOqZKQaSCpaVaMoUbjPXmJaiuqfrQOt3QNhsF1VX94mZ0nHAnWfUz8ja7r37kPjYW3Sta6Fva2I9Po6JvtTc4yCWVT0kkOsa8I9+9Mt1eQDNbKMzCRZNv2BxThNhwFfY922kYnTdiX3TUsBzvwEWXd/17zjRF8MwDMMwDMMwTF4G3FP2pJNOQlNTE2bPno36+npMnDgR8+fPzyTqWrdunU1F3n///fHEE0/gyiuvxOWXX44xY8Zg3rx52H333TPLXHrppYhEIjjnnHPQ3t6OAw44APPnz8+ECC9YsACrVq3CqlWrsMMOO9jKYzh1mBl7hz6V5aFr9FIkUVxS2HUSZd1eIBGS4k64Hli32BRlO0m8AIB9ZwLzzuv9vgxGkpGtXQJJevv2UB4w2tdDffH3wHkr7J+nEySu9gYtAUSa6XfpOEUvA/bBE7EcADP7lvmap40TApmWpLpn6FRXtTQAheqpnpZRvU7rSsUBTyBPoR22m+i5l86gwHo8DCdRVgHaVgMrFwBjLMKrrgGeIImZ6T6I9F2hekh4t3rVZiJZXXSek52AdzNE2XSCxF8z8Rz0NEVydxcF7XSMGIbZPkgnACj0TMcwDMMwDMP0iS0SK37BBRdg7dq1SCQSWLJkCSZPnpz57s0338Sjjz5qW/6EE07AihUrkEgk8OWXX+Loo4+2fa8oCq677jrU19cjHo/j9ddfx9ixYzPfn3XWWTAMw/GPyYO1Q58TKav1PlI21YUo6/IBsTa7uBNrI1FWT8nlXd4fTtRmcguJV1qKjnVXsCjbN/IJVPmO53+6GHBIx4FoMwlpLi9FkgNSOM0sZ9aPTJSrQ6QsQCJsJooyTZGyhkbrS8dIiM1et2O5YnkiZfMkr0qE7O+fOJkiOLtCRH5uTcKNJIpnYxUZHffDPA6bPrV/rCXJtkBLZuwk5Dr7eF+6poR+63JTvba203raEhGtkajv78YbPdYGPH6CfL9gtixbOm5Gypr7LNadiuWuxwpHyjLM9su/TwEW5ibIZRiGYRiGYXoOG3gwuVgjZQ2j99OJVbc53dXj/Fu3Fwg3kAgraF8HBErpfxGpp6pwFJi2R7ZUROHie4EHD+96GScR8Ytngfb1A1Om7YVssU2QfTzjIRL8Pvt3/nWJaEpvAYmgIlJWybIYENsUEZGKmlVlDLkem32BGSmruoBklN6rbml1kI9kpOcRl6o7V5T99tX8AxDfvAhsXEoidL5juaV4/hzg5T/mfm499oaDKCsGt7JFZS1Bg1BOouxtuwDrlliWTQMt3/WsnJ31GU/ZTFsrxOJ0nI6lodH1lM92QpCKAyv/J9+/d5fc33SCtiNEWt28hrqL+nU6RgzDbB80ftP7WSAMwzAMwzCMDRZlmVyskbKfPQmseJnEnp6iqEAqSoJAvkhZAPCX0usJc4H2tVKkVXuZGGgwk4wAUMjzcUsQa+8+GjLbvgIAnjsb2PTZgBRpq/L50/23rnwCVXa093/OA56f2c26zHPgK5R2H0Cu72tGlFUt9gUOQpgYIBG/ERGUqsusqz4aDOkusjEZAbzBrpdZ+Trw5GmAr1hG+FrJNwDxxp/pfPREHB5oFNV56r312DraF9gTsGVIxaV9QbYoG24AWr8HGpfT+9bvgXv27lk5U1F5TYjIZ8PhPDtF47avs9dpp/2x+oy7siJlXZ7uxXOOlGWY7Rijd8+GDMMwDMMwTA78NMXYcfnsopzwdFXyTE92QnXROtx5RFm3KcoWkq8wKnY2I2WFKDvgVsfbDu3rgZrdgeQWEmWtvpD5EIKglsqK+NsOo5afnwlEW3u+/MZl+UXFvJGyWVO8I40k0HVFJlK20LT7aKf3IiI2eznAjHw1Bc14CHjpDwAU4JQngcoxUiATUY6GKdZpKaqrTvYFimKfpq9r+eunEP4avgS+fwvwlwDv3527nJNQC0hh2eWhMm1NVJezkGmzL3ASHEWkbNa1IBKkOUXKAsCnjwN/nwJ890bvkv5pSWQsK0RCRmt0q8tLQrqTD/BH/wCePsOyPw7bNbIjZS2esvmsMgDglUvoGuBIWYbZflFUHnhhGIZhGIbZTFiUZex4/LkiUm9RLNF3TqKssDMoMZOwCXFWiLJWb87e+C1uqWjT/iQRAoqH9r99wbolzp0lXes+skUIfe/cDjz2M/l5Mtp/5duW6M4X08pDhwOf/pvsB/6yo/27fJGyWhp47DhpQaCne+DdarUv8FFSJSA3ElYk+hICqxBWG74EPn6Y6vIuRwG+oiz7Ah8tb+gy8ZfqzhVDhVBqxalOWgdtUjGq4wWVzv7F+Y632DcxHX9Lse4DYO37WWXJIzbY7Au6iKTNti8Qyd7ScWefbHEu/jm9D6KsKFvanghOWBak485+xwVVQLjJsi7LdnWr+Aq5DmukrKLmXgvLX6aBi1ULyQqBE30xW4oFV5P9CbPlyB4kZBiGYRiGYXoNi7KMHXfAefp6bxCJvnyFpkCTJ7JOeFMGK+h1cyJlDQO4aVjvf7e1ScdJvOpLpGxXyZAePgJoWp77eSravSdoOianb69bbH6o0G+3R3qT2MxbRMtrKSDebv/OFEjVSAPZflj9N79/E+hsoPeG3n0kqJMoq6i5Imk6aUZHmteCiDIVAro1CjjbU1bXABgk7Lm8zmJovmn8XZY9Tr8rGwWMmZb7fb6I4kykrIM4PJC8fw/wzm1ZZXF1b1+QnQDxo39YIkmzyp+OUeRwMpJn/y3ipvjt+/cC797RddkzAq8io5gNjc6llqAIaC3p3Ka6vPZyWs+9+FyI0Km4GT2sm9tKmyJ6lii76Abgi2fkwBtH0THdkYoBH/xdvt/wMfB6H5JHvXcnzbhhthz57HIYhmEYhmGYHsOiLGPH488VqQ65rHfrEJGygTKKDsyXJExV7a/CY9YqIPTUNiEZ6V0ZtxXSSRKl+xIpe30FZYnPh9N38RCJ5V2WKQEEyknQE9PsvQW9iygdDFinefcUBcgImdmYApmn+Wuo/z3f4uFqClwiwZWITu0KUQeDlRRxHm0hQThblNXMCEyxLREpm45RJKQ10ZYQ3fQUrdPQSQsUkbPWKEtBtoct0LM6KSwRnKKyuxJlhdC5JSMsXQ5itOoQIQxkecrq9rbt5Vl2P1dAtktaijx2k5HurzdxvDcuJTsDAHj1/5yXtUXKavJ8uX0yAlpL2sXjnO3pNMvAJsqm5avqputR2Beo5vlR8iRiFMkhFcVcrhdJIn/IhJu2vm3HluKLZylJHUBC6nzL9d2xHnj39r6tlwcBthw3DafrlaPhGYZhGIZhNgsWZRk77oBdlK0eD+z/+96tQ3WTgBcoIxGwJ5GvqpsiyQDnqbbd0d/T/7cUIlK2O+uFfJ3NtjX5fxNz8EqNtsiI5HykYkCwHAjX07R3gMTZ7EjZuk/tfqODDSGA9CZSFor0Yc1ZH4ltutc8ZiLaMCPKmgKdASmmaWlg6VyKMLSSTgC/WQSUDCNxM9pKYnp25Go6KRNIAVSPYm0U2VhYKy0TACmwZewLhCgdp/dO4qTiov3oTpAXScasx8LlcxZw8wlPQozNFoJbVwOPHN319jcH1Z0bdd4j+wI9V3QW+ybWJzxb0wlq31JRaTlhK4NFuBTnQLF4ti75u72MIkLVKsoKAVT48qbNRF/ppFz/dZVA80rLuhTgu0XArWPl9WoY0sM4bSYos9oXKC6z3ptRcvcfZLk+zDKLsuhCxN0O/aj7mzv3AL6at7VLsWV47my6f7xzG/DJv+zfuf19X+/2OptjWyQRAjrrOFKWYRiGYRhmM2FRlrHj8dsFmLKR3Wdbz0ZVTVG2nKZ490SUnd1CQiDQN/sCERE42CKN0nGKhkx2IypfVw50bJTvOzbQa6S5i3U7RCT2JMoxHadzF260iLKBXGHugYPtkZiDDav3Zk8RQpmTsGYebyU7QlbPEmVdbinCawlg9Vvka2pbV1wmxHP5gGizmfTLYxfntCSdG6t9wedPAd+/QfXJ6gMsEoFpKfM3pgCajNA2RJRtpEX+RnVRlOYzMyz7n+/YWG4nWlKWPyMgCvuEPMdbiHeq297Rb14JrH0v/3Y3F8cEZz2IlM22LwBkHRFiqYhWF6JsMuw8CKBaBqJ0i2BqFTOtArvV29X6O7EvLq8ppJoRsxk7gRRw7z7AuveREXx9hUAqIgXnFa8Ad4w398e8Dm2Rsm4ZKWvowKbPcpPlCcHa0KgsLNx0TzqG7TKZYj7cXvIe/vwp++e9GiSz4CkYvIOzgw1rfecBF4ZhGIZhmM2CRVnGTnakbF8euIV9gUj04ySyDtuni99bhJ+ebl90xgaLjcGGj+k1nSBRpCt/WIHVw7RlFTD8R7m+pgCtS3Hl6dz2RJRNkKVCZ3akrEO0ZF870NsC+mZEyjoJuUJsFK9WD1dACrEur7xe0wkpYD1/LtD6vSyTEDXdPqpPPlOUtfqACvsCEfUq6lr7OjO5l1hWMb1jfXLdIqIyGTHtC9xUnlt2suyuyyyT0fu2wOWxR5xmooO78ZS1euQCAx/95mSXYBjOArR1ORHVa0VcS+K4i/qpJQC/sC9w2H9rGymul2zPVmtd1x0Ebqt9gcsrhXEn+4JYm9xHIaaL7TZ/a9mfmCnspiyiucviKavnlg0w122ee5eHp5X3lL4MSA5WkhG6rrLvK70ZJLNSPNQ+cMkMHE4DRAzDbH+kk3z/ZhiG2QKwKMtIDJ2iYjfXO1Qk+iqopIgKJ0/ZmQudf3v2gr5tU0RsDgZRNtYGPHS4Ke7FSQh34h9HAM2r5PuoJYIxHgLKdpTTjK0IAUhzepjqgbCWjgHBMiDckBUp6yCODebpor3xlP3zDpZrK499QToBQ/VAEaKjnjYTaqUAKDKZm8triexOmgMhCeDzJylzfeZziygLUKSs6rFvW9gXZDxlzYjLeDtQORaYfJ65TTdFPYrz6PbLhFBWUTbcYN8nkVxMiCeePNeqEyL6NuOzmjDFwnz2BaYoq2ZZB/RVpOkpIsLUhgHHAQxruXQNOfYFmUhZUzSNmeJFOmHxlDWF25WvS/uR7AhcINezNZ0gL85Nn8tlbOVJSxHW5aHlXXkSfWWSOSp2/1h3AHj9Gvv+uL20LtVN5ckI7Zbjk7FfEcntNJkYTPVsn76T3XVUN3zc+85sT6/1lX28T25LJMLUJma3B6J+ZEdfd0fxUJpOzxBLHx24dVuf6ViUZZjtl2fOopl6DDPQ6Prg6MMzzADBoiwjMTQzMcxmWgAoLlPYqyRf095E/wzfL2tdPUz0leikxEZbu0HvrLcIHnkQwk0yTJ1wEbWYzfolQNNyGS1nnZoZ7wBKRzhHygoBKJ3I/zAV77BPU8/+faDcTBgUoBul1VO2aYVcNjmIRVktRdd7TyJlk510bkXUn9MUfC0JePxQMhGzpk2AliZBNWNfYEbKiqh01fRtVVQZgZSOS29FlynKegKmfUF2pGzQYl9g1rVYOzDuWODIm+i96qHzJ+xJXF4Z9ZgMSwE11mbfJ1WV5UtGZOK3rhBlcXvtycN0zfS/zZfkTJGRsjbf3AFOMOckyuaNlO0i0Rdgty/Qdbv4LuwLxP7POw/4/GnTUsIa8ZrHvkBLAG2rgc5NZqRqlr+woUubkUykrN9uXyCw7ps1AVxhtX05qwWC8JQVib4MTR43q3+yyyOjfIV9QV8ibTYug//bF3I///L53q+rv9n0GXDL6K6XeehwYO37PV+n6pEDN93x+C97L1puSygq1WvxrAAAX/2HXoUw/c+f926dbn/3994fEi9eOHDWAtaZDCzKMszgIdYGrH6bBsVWvAo8ezaw5t38y4t7UioGNHy1ZcrIDC4al3dtpdcVsXb5fPjxP4C//7jfivWDoiezbZltHhZlGYmuyWgoAFj/IfDtq71fj+oyp8CX57cv6A0im/cbN+VfJtZOkTI97dQOFHfvDSyb2/UyQpSJtkrxze116FAqlGwr2koCXjICfPQQ8N5dJPaUjnCOlE3HKVI2W2xMJ5CJbnvnNmDuT53LJxJ9+YpI5El2kvAnOsv37Qd0NshlByu6KZZ2F50mxNRIsxS7xDk0siIZvUVQxPIi8lBPkcVAJlLWjBz0F1Okq0imZBW+hVgP0LUBgM5dllCYiZQ1b8iirkWa7FGtLo8Z9WiJlBVTy5MRU0B1S6FHTLFXXCSwaSnyHfUW0u//fkD+4yWOjctrWhFYojrd/vzHWwiF2R6vKYtAPRA42RA4JfESn1t/l72MEJj0lClUmtdHWtgXRGW9FFHLqttuaWC1p7BFyibpHKZipm+sLzdS1lNA63R5zLZFRLlmJ9uyJBGziuYlw+37IzxlRbStNdGXVczOJDgzLTKEGK9rNFDQB+FG+fo/KFx6X+4Xz87Y+t7h4cbcAYz5lwEfPmj/TNiR9ARvLz1RtyVRNp0AQr2IUhV2OB5LvRYR2uk4UGzOTOhKVIyH5DbrvwBWvtb1Nuf9tu8dx8FK9jXaX2QnPARYEGeYbYklD1A+gE2fA0+eJj9f+z4w91hKLPnvk4Evn82fMPi9u2kQGABurAUeOLRn294eZnIwPefzp4AFV/dJGFTu/zGCXz1Ob1weCjyo/wJ48PB+LuR2zvUVAz+rkBlwWJRlJJlOtimOfP9m39YjfA19xaZ9wWaIsiIxUSoKvPWX/MvFWklQ2JqRsisXkHCVHW2WjRBg6pZJ8a1slP3BKBkhkTkeAkIbgcoxJOqteJXE8rgQZR06XekE4CsB2tfK9wBwQzVQ/zn9r2tSHLrvR/YOvoiU9ReTGBtrIzHOGuEYrqfXwWRfEGmxR/YKsbS7SFlxTYkM9FpSnkPrMdGSgK8QSsoiynoCMmLWlqEeVD+0BJXJE6BjnAjbxXogS5DMiuAUlgACYV+QzhIyVbcZKRuQYoiwL9AStKzLIyOvhbiouqi8WkqWMxkFGr7If7zE8bQmDwPMSM4eROKrpuhnXZ8QqAcCPZ3ruSo8UXOWtYqg5tR8a1kzycxSdmFZS8oo4XSC/nebUcv5ImUVNTdSNhU3RVmNBFfrNnTNPD8RM9FXQvrBZu+fEExVt73Me54AXNVM7RFA14E7O1LWPKdayhIpa7FAUN1yGUM3o7L7ECmrJWFYr20rWyOh05p3gfsPpv+FlYz1/NR9Cmz4SL4vqKa2GzCj67vpsPiKe7dfqa14r8vmi2eok9+2tvtlMzMvYvY2qtO8p6QTwEmPASN/LI+fE38ZDsw5kP5vWQUccnnX2/30ceDJU6Wf+/aMaKd6KsqKKOUer98hUvbGmq0/U4kZ/Azmgf5tic5NlLSz4Stg+Uvyc1F3Y61AyQh61hdtbzYLZts95kdMptfuxNnHf9n3cjODj/a1wKf/omeAniCemx4/AUqoDsXv3UjWaeJ5snklsPFjYPF99Mz76p8oTwbTNSIBODNoYVGWkehZEWPewr6tR0w/9hdTxN7mCCpuH/DKH4H377V/nuik0TRBrG3rirJaClh4ncxQDpD/4zcv5XaMtCQwaQbwzYtSPCvfyR5Vleik/Ym1kXhbtQtNbU90mp33kBlN5BAZLKLyPvkXvbduXwi0nfVAYS393/QN0LHesowlUtZbaEbq+u0CRLiJXq0P0N++RqOl2ypP/Qp47TL5XtdoH7sbXUxFTfEyKT1RhRhrFXTTCcBLoqzh8uWKsum4/Rj6iy2Jvlymn3MEuHkUib5CkBKvYrAjOzrX5ZGim1WwtUXKeqUnbCpKYp0Q2MT3qkuK1qmYaW/gNj1EU6Z468uK5HRAHE+3l+wPRH0wTDuMvPYFJopqj6zUUnbhub/RdYeBoyxPWXHMRbmSEeC7hTLqObMuyzR+q3Ah6rmWMkXZgEzgprqyImWFp6ySdRyStHwqKiNlRQIuwIz8NsUuYTngFp6yrtxy6mnTv9cqqHrsiblEpGzajJTNeMVqzpGywgdXNdchko5ZheueoiXzn/etMSOifR2w6VP6P9oClI2kSM3XrwFevhhY9749WnTIBHmP+uBv3XdUfUW9269tzTpGTwN37dn9cskw+c2HG4CiWuA3pr98oJzq2UcPUV0pG5W/IybqS9SMfG1fTyJud3ZH65eQrcT2jrBNiTRnBJeid66zJ+iy8sxZ+e2MnNBScgDQ2kZtiUjk9R/Ss5VgxXwW8rYHPn4E+ORx4OZubGGYXLQ0tZ3WwA5/MdX3zjpqWyPN9CfuG2OPBGrGUxscbpS/e/Ov9HyfCAPD9gb2OFF+5zFnzNUty18W8exgGPnbG2b7QgzG9tTT/flz6LV1tfyseRXw3Rv0vwgOee1y4IkTgSVzWHDsCjHg37a66+WYbR4WZRmJ1SMQAIqHAEfd0vv1iAQvviLqHORLZNUTXD7yEMyeBvrFs8AcyxTqRCc9XGwt+4JkmB54CirlQ8/3bwD/uwL460j7sloCKBpiTmU2ha6S4XZhNBUFSnYAPvknecPtuD/dwJIRiu6Mh8ijMnt656IbpXALADsdYhdlVRcJeIlOmcQLANa8J/+Pd1D5fMW0jUiTZQq9iRjVtEbKrvtg2/B7zIevUIrJAHUsvQXO/rBWUjEgUGb6v3qkKCt8ezPrSwLeQqipiBRvPQESWn1F9Hs9LX1fs3/vCcprp32tFBhcZufX6s9q3aYQTnP2t1j+7/KYUY9mdKbbZx+Ecfto34SHaCpG28/YF6SlUNxdpKvoIGciZS32BR5/16Ks03Rlx0Rc/YiedrAvyIpItgrLAPkqr1+S65dq9WcVD0oiUZrHtLIQkbLiOKtuu7hv9ZTNti9Ix+V1JCJYFXM5Q8uyL0hYPGXd9nJqKdneW8ucHfkqBHwtaQqtaSnaWq8fIUYriozA1dOmp2wfE32lu4qU3QrtfKSJfNJTcRqoGjIRuGM88O4dwEembUG8ndrpT58wo9HN89e8suuoRS0FBEpz9yu0KXfZ5a/Qa19mKbR8Z3/fG3uFruhN/RT+7y2r6B63wz70eUEl7VO4ga7boiFy6qzTOsYeSX8A0LISKN0x/zazBwUeORqo+8T+2fYU5RkPAVCAJX8H/jYFAFDw1eNdRx6v/F/P1y/aH8DerkR7Iez2lZX/A167Qr7/8H66lnjq5uDmpYuAz/4t7yWvXAJ8NW9rlmhw0LyKpi7H2oC7JsjPDYPuRytfp6TAz59DQRPJsDmIr9DzR+kIOfMNIMG1Yx1w0zB6f/yDwJF/pd9oSbNt6QLRjoYbgb+M6M89ZbZFlr9MUa0APRMBXdsOPXAI8MXTdE8u3wkA0HHIn6GseJmsNAC7mL/6LXp97Yrc5xeG6v31FfT/l88PnI88s0VgUZaRZGcT11JAQUXv1yOi9nwl9Gr1jestQgDIFluzo+eSUbIN2Fodq2SEOu0FlVKUKqiWI4hGlrji9gIwZASdNUoq3EgCr9WeoHYC/S+i5xIdNBKup+3rfvtmYMXL9B1A0UY2awLhM2kmmNLSuQnStDSJsb4ic2rTJplsShBuAIIV0uMScE4otC1hTcz02hUWT9nuErNFSTARUaliCrqv2B6hk46TfUEyIiNKhdDqK5ZTz8WUXeH7qyjS0zViRiw0fm0pgMVr1SpupeLSs9lab099ml6tkbKq2xIpa5bB0KXg6/ZLcVAIyICMpGz8ijrcQggE8idvyvGUtdgX9CRSFsgSRLXc6ff9iZM3bHakrBAahTgrjlu24OhkXyB8dEWkrCYiZa32BVYbDLNTmu1rqyVkpKxIoKWl5DK6ZkbKRuV1KgRVJUvQt9oXWD1lxYCBiHgQ1hFagr7TUmaUtdhnxV5mEeUsrptMpGxf7AsS+UXZ7gZSBoJEmKJjY61UF0p2yF0mWEnCwrzz7YMiQsx+7y6KRALs0UnJiNkOZ9kX3L5rrqD45Cn02pfowHv2tr+/e6/er8OJntRpQaITKKyhyJdgpfy8ZAe6rwB0zRTVUpRnpNlu/dDZAIQ20P2xbS3w+rW0XGmWH7KV79+wv1/7HnX8vn4BeOp0+uzPQ3u+D9s68Q66VhuX2z93EvkF887r+fqFHzlgf/7YEnZGimpp83RKute2luyZONnJtssLv5O5CJxw+YA17wBVu9L7Fa/KmQlMfjL9hqxjK9rkde9Te/DdQgqSSYTo/h0oI8udyjFyQKPxG3reF/cmEVDzo/OAq1tp4Kt5BbpE9CX6697CbNt89A/5/6ZP6fxfWyqTwon7Q6ydgmLEYOj/rgSCFdBPeAyxMceSULujGWj1wRx6Pelfct11y6T1UFcR2KkYcOsum7lT2zDZM6SsffdPH6e+Y1f3eWabhkVZRpJJeGNJTtOXacNC+BLC4OZEyrpNASDba09V7R39ZJhE0M0VZW8d27ffJSMkwgUrZcfEE0DmWCYso8tiWq6iStHDVyijpOb+jLycPAHgT6aoWzHajE7xkyChpeh3NeOBb8wM5YZBImDTCikIlI+yR2ilYiREJsN0fte8Y4pDZpl1nSIAPUES54Km35SIlDUMEnjCjSTYpqKUACDSDGxcRh3jWNu2lYRGYJ0OvfY9GSnbrX2BiJQVoqwZKSvsBwQZ+4Kw6fepSbsAfzFFqgpxDjA9ZZPyvCkKHbfyrOl7JcOAS76XSbOEYPnmnylqKFuUHX0YvVqFTZHoS5RHiLI2+wJ3RljOCD6KZWr9mndkojOg+7rm9tqnzWc8ZXsg4Fg7+rpG9X2gcIrEzfaUNYR/rLkv4rxbp/qLdQEAFGdRVk/TQ6knIM+JU6Ss4jIjV63bTGZ5yprHUlHl9oToLjxlbYm+siwhxH7rFiHZehxqdresKyktPNwOImvGB9cl7TgMTR63vmRoTycBNev+s26JLOuWJhmmiKNoC/1NPhfY82T7Mm4ffectsouyiU5qaz57ElhrZrq+1dIZTsVo4MdJVMo3+2NrCNP5sEb4fv5MN8t20gBqqI7aVQD44wr6THTk3T5TlN1EPr5fPid///IsSlzjCZL1zrt3yLq64eNc4ccwnAcLY+0UYbnmXdnG9WXwYFukaQVFIHesN5+hzPY0tBG4tty+7Hdv5P6+O/S0vJdaB6Ws9jcDRSpKnfKVr9OzSrQFeMpMZLQ5M6U4wmjgWPEqsOwxqq9OpBP0rAoAxWaEZran+g+NSDPwn/O7X05c82/dTK8Lr6PjZn2mOOVJoGYPOs4Lr6PPhuxJf3ucSPU5VAf87Uc0yCGmik84yb6t8p1oIKsrhGBWuTOw60+pfb6uYvtpWxlJtBVo/Q6YtRw49i5g/HRKDgeQnR0APHgoJZf753Sa9Sn44D4ahB5zBN1LKscCB1xE34Xr6bORB9h950Wui7+MoKTa2cw5APh2Pv2+u4juwcqfh9hFaWv08Jhp1De/fdctXy6mX2BRlpEYWdFhXXn6dYWIahORFJsbKWvouaJsOmn3qtXNkd/NFWXDDX2LthAPRtZIWatgJ8p/y87UEXR5Tb/WFrrpeAssCaViUoANlAK//YBeDSM3e/2hVwBPn0H7n+gkUTgeskTKjqTILgAoGgr86HyKgo13kKiw8WPpdwrQVPvRh1Ek05QLciNl0wmK6Ao3kGArvH86N9E2PQXA/64iH6BtDc0ivsU77LYCXZGKAf5SmVRLS1rsCyydT5HoKxmRfp8i8ZK3wJx+HpMRrP4SOp6KQp8ZOgnaRbW5ZSiosAtogDwnimIXLV0e4A9f2X/v8kox1mpfIMQMt4+Ei5QZKZuM0HeqS4o/7evMqElTEMvbARa2C1n2BZlI2V4Kav0RKXtTF1F02V7agCksZkXrWhNWZSwaspJYiamXqkv+LwZSXG6qw6tep3olImSdPGXFuXa5KXmCy2uJlBXtg5e2objMdSQs9gXmdSqic7P3T3jKWiNohaesYJ8ZdD1aE33pZqRspoNldppzImVNcdYwo2/7ZF8Qh5HtR/7wEeb2LMdr6dzer7s3bFhKQl+ikyKFoq20v6UjgLHT5HLXmA/K0RY6DoFSuvc9N5MG5da8S1EM/hL5G5FgJRWlcwcjd0ArXz3r63Ttvvj7doe1jIuu73rZRMiMCg7TPQSgNk/Ugf1/TwJtsAKIttG9pmWVFGgCZSQgeIL03lpPI43m1EgNeOPPwNNnAjfvRBE6x/0NmGjJQv7MmTIaXXRyEqH8SW8GE6ENwNC96BgPmQC8dye0YDWUUJ3pB22pj21rSLQRVhBW8l1jumWmknXA5b07KTnLjQ73sP4iGaV79+PHA48cCUy91vKdQ10RfpvWyPRsdI2iu/qLhq8oger2ztrFsu1PxfIL/Mseo1dRt+q/oHZR8MrFdJ2e+jTd59IJaj/jHZvnSzqY63LrauCzJ7pfTvg4f2Vah71zG0UhppPAtJuA056lnBTnvwusNIWyn98P7H0mcNZLJMzuczZw+zi5zsZvgAmnAHufYd9W+SiyCwHy30fE+fKX0r3um5fo2cLqdzsYSUZ6HmyyPQ8mbPoc+ODv9H/rarLIKx4CTDqLxPy3/gIM24fu8QAFITxyFL2KKNlKM/jJ2o+v2Q2o2Bn4nelXfOrTdK//01rg3Lfps1gb8LB5n2r9Lre/X/8F+aMDlAh0e8W634/9DJh8PnDYVcCEk+25dpxY8sDAlo3ZLFiUZSSZzo1lSmqfImWzLqu+JgwDpACQLcqKjPEZDBK+3rgB+PTffd8e0Les1qKRDFbK3wsxq2S4HLWLNJFI6vJSRGKkmW5K3kK5Di1timPmcas2H5b0tCnuWUREt49Gujs30V/5KHooEn6xFTvL7KlDJgA/uY4yLbesomlL4SaZCfuVS0h48xaQkDJ0IhAso6kQ4sYpvG471tMNs8P0qEuEkfHAFKJtf6PrXWfgXP9hz9cV76Ay9yhSNmqJlDXFrkSYRAOHSFk1ZbEvcFsSawEycRIgPWVFpGwqRss6ibJAbnR4xuc0a5o7kDu1Ws2KlM2u125TQE3HqFzJsPQfTiep4xttldPhAfuDgdNDqOoyk4RZPWUD9igqLZUVVWWuxyq0ZETKPopJhmGPVM9GURzKn/Xe0KWoClgiZbOSWOmm9Yc1gtZjDqQI+4ldf0rn2DCjnm2Rsor0dhWRq6GNps1GgpYToqzLS22FotL5S0Ysib5MixJxvSqurOhjYV9gFWWzImVLhluE+KQsk5i9IMorygxIUVZR6bjoet/tC9IJqNFmKHOPzf3OKsq++PuedYLa1/ets/TPn5MlgTVSViAiPa2kYnTs/aVUl754mtrQmYuAM16gNkAIGSIxmDh3yQglJLRijUK12eB0M5iUj+zf9XQQUktRedvW2qcsAvb61b5WdpycEJGyAA1OCMRg1WRzGr2/hAb9anajSNlrS6ktUhS6/rymKGsVBU//D11vzd8Cb/0V+Hoe3W+f/w2tb/rf6J5YPd78rXk8xeBiqA64bTuY+hgPSY/dgy6F8uZNcEUbgfY19JlVvIw00z3bMOyJV7Q0WQI4YYuU1eVxXPse8N2ift2VHFIx++yrYDlwyOX0v7gn3ThUttGPHkPTOm8dkz96qr9zEbSuzh8VOlgJbcptPx85kvyyARJc/jk993daClhh+mDXfwE0fQu8+RdqF68tBx4/gURbT5AGub5bRG1uoIyeU/8yondJfv4xTW73tl0Gr0AmbALCTV2LgRs+yv1s8b30LDf+OGDMT+TnP76QAjUmnCzbT0Bav8x4FTj6VmD5S/RsnI0YuPGV2AMSrIi2NFAmn4WALeM3PZA8M4OS8HYlerV8B6x+G/jvBeS1uj2ybjEF3gB0rqt3k9/tZw60FA8F3r2d/rcO9HfWAee8JaPhLdeEceRfqf9aMRo47TnqrwL0/DxkAjD97/TssW4xPUMD0nLo1f/LfW7anBm62yrCSz0794DLDRx0MfUrFpjnxqnd69gIvHqJfP/qnwZv+7idwqIsk4WlgoroqN6SPRXYX9r34rh9VKbs0fJ0tigL+RCx/oO+bUuIK72Jtn3gUNPzVoiyFbmRskMm2EXleMi0LCii/VJdpihrLuMrIqN9a4cVoI5v0GEaUPkourmFNtL0omgzHfPiHcifq3mVuaB5bn/5iLmdYukNm44DHz5ADxTWhzFfCU0FEZ+lYjTq3ricomkjpqgbbpDLiGzt2YSb+h71sOwxSkpz5x75l7Em/3BCCEaGQecgGe6hp2wsy1M2Reencmf7b7UkDG8RlIwom5aRsi4PiQnpuHxYKKySwr2IYE3F8yesEaJe5iZqvjqJstm43NJLNhUz67WlrouoVhEpmwjTexEpW7s7iRtukcAs2HVH9ti76Lq32Rdo1PmPtdL1EW4ij81//py+t0WmWsombA+E8NdbspOjZaNYymjdvqJQpM3GpaaAabUSMM+7vyQrUlYD/vA1Dc5Y7QuEMCqEcUWVIr/qlteByyujYK3CYybi1W2JlDWFf0WR4r/HFPaE5YBVULW17Wm7fYGwxhCeskfcSG1IrM2S6MslB+qsbZAQhwEpcGfsC8wI46WPAM/9prszZUd1QY21QBHT/a1korPM8yA6gl1x5+5kwdFbhK1POkGzDZxE2V8+bFneoPMZKJNCY6iO6sNOB9MxT3TSNSASL4nrQrTJVqyd31SUOjSHXO7cxvYEYVUj/EZ76gd7fSV1fDo35SYIy7YMWLc4/3qSYTloGLR41h/5F+DKRrJrAeTARtEQGVmTFIN/kJGygDzO4lp1SqomIpTFgBpAA6iJTjOhmJK7X9HWbWsapJbuOuJTkAjRAAIABMug6A6DAOkEreuNG6hdbvwGuHsi8P1bdNyt17lh2H3qrJ6yumYOSJrntC9WJT0lGaXowUu/k8+VwmYJkPekVARY+ihwTQnQtBxoMD3aW1Zlr5EQz2fXV5HP5uYmmBKDFNvStbM5hJtoSuyLv6d79jUl8nlZDJTnO7bRFmDHHwP/7yMaALhvXxL9AGpbRYK5aX+Wv2lZRfc00V73VJR99tf0/N/wlWxHu0qwuC3TvBKAAty3HzD/MudlVi6gfb1gKb239okSYRnYIRDWQtmI+9iO+5OwttMh0r/TissDnDAXGHUg1UXRnliJtZP10bBJVEdDdcCog7oeGN8WcJrBpWvA50/TdVVsCoBzDgBWzM9d1jCAjx8G5h4LfPov4MlT+xZIkIrJNv6Tx3v/+4FGJKzVNapbgVL53YSTgeGTgXHHyuuseCiw28/lMkMnynvERss1Zu3DjJkq23TBTofKACNFlcLsmvcooeU3L9Kz0cxFwMw36Frenrh3XxpsAagNtQ6mi2MnxG6A+vKAPE/t6ymC3sqSOd3rHR0bug6GYvoVFmUZC1kRYyJRTK9XY7npV47dPD9Il4fWlz3Kmo5TR/yr/4iNSlGwOxEmHz31yhQYBpmPx9rlb6x+nOk4PezU7kEPJOEm+jwqImWLZGfA7aPvv36BLBDiHblRWPEQddpXvGKPRhHr6dhIo4zhRuqAzvpKTiO2TlUfvh9w8SoSJDvrpZDsLaIpttmd3XCj7ETH282Eap3UWY61kwDVvs68iSq5iYsEz/8GeOkP9s8S4dyO8BfPkj+tlRd+172HYkc3kXAiS306Tg8VwuexN5GyQhCItpLonbKIsoYBePxkX6BaPWVjsgMrPIQBOn7xDlrG45dT0/0l1IHJJiOgmWUQwtiog4Fdjup6HzKRskFZHquXqttHn6Vjpn1Bp+lV65JWDUKg0zUqY1f1ZNJZdJ1k2xeI+vHFM5R0yOWRCQGsRJqAjx4y9zPdN9sDgdWTda2DWKS6HCI5TSuXr/9LkTeGbrcqECKHv4R+Kx6+rYm3RHndPhkpK4Rx4Setuu2DWOK8uHy0TTGI4fbS1EMRTa1r0iIjEykbtovuwn7GmoBPtOdiCrNI9icGG0RZ9r+ArslkRArEQmjNtmxQ3VmRspq93XF5qJ1w6sh0RfbgnvX8i/YlYrap7evtyzpdm6UjqMPU28gAIXQDZCNijVoS7eLux5vlaZadlECp7CRbE3gVVtFUzsqxphgIEpE8BVTGdJLaenFNWdunSDNFK406yHkw6bZxuZ9lI0TZ/5mDWL3xpm39nsqQ7RnaE+GjeRX5CyY65XErHSG/d3vtlkQA+Rt++6ps8xKd8n9PEPj9pxT1KtYnrnen8y/Oi56WHb5QHbDjFLouYMhoG3Gf+ufPgUU3mL/Tu48qHghrCCubPiVRLJv377FHcMU75LE1xcuOg2+Qz0aRJoqCffQYel9QRQONAE2HbPxa1i2A7u9v3gTcfxDw6E+RSVwH0PWUjtOsGiA3iqc/EVOgvQVyIMZbJNuGdBJ4/Rpqq169lD6Lt5NljGnj4Ih4DtOSwAMHk7XF5iDa7SdPzf0uO8p8MCAGWTZ8LK+z9UuovolzUvcp+ZZmc8fuFMFVMdouzFWMsc+is0ZuZtdf0eamk3QvyeafvwA++Zf0nm5fL60LehNl21Oynxkf/Wn/b6NjAwCDBrGtAlUqLu9hj/+SkhiWj6L7yZUNwNXtwC7HUJucPUuxeAdpbWalMGt21j6/BiblqQO7TaeI5pWv2S0PBPEOmpH349/TM0D7WrKN6a8Bip4MSvWF6ytzz+uyx4DnZ5I/arxdRm/++yR5HkTb8dFDUjSr2pXsIbqaMfj+Pc6fL5lD7SwAfLwNthVihsSadykyuGyU/fuCKpqpVzGanv9qdgNOeJS+O9QUBUcfBuzby0H6QCl5pQN0/xl9KP3/1l/kMjPfoMGAYXvnn+UbaQE+7YEtyLaGEKTFs2a4gYRaABh3HL0KUfawK0kj2PQ58J/zgNvN3DNfPU85S6zPb90FNDx1ukxOyww4W0SUve+++zBy5Ej4/X5MnjwZH37Y9TTjZ555Brvuuiv8fj/22GMPvPLKK7bvDcPA7NmzMWTIEAQCAUydOhUrV660LdPa2orTTjsNxcXFKC0txdlnn41weAAfFrcLTNFKoKVk5FRvsPoXXuAgLvUGl8+efMw6fTgVlf4xMOQDSMqhs9oTRCe3p1PZREc03i5/Uz5adnrTCeDCzyh6NRGihxNA2hd4LaKsotA0pKdPl+JytiibCMloH6vo6Sum9YTqaPuGZh+9VFT6rTg+ikLCgNtH+yCm4e8wyYyUzRJlO+tlp7fxG5kdt6hGPjC2r5PRR9YoGiueYG6H7eN/UCIXgB74HzycOu6fOowQd/dQF2vr+tyJqfSiwyQitrqLlE1GpaesQNdIYBRRbCIBkctrj5QVSdRcXrp2RUTcgRfTeY62mMfbFBrSpnBaNdah/FnioZ6miNQ9fgkcdEnu8lZcXnPbZnlUi2gHSE9Z04IB8ZApGKrUERKirNsU5PwlPet82zxLLQMDMOQ1abNB0OlYhOqAl/8ofyeiQp0QFhr5ENuv/5ymW276PLeMOdFdZlvoK6LtZuwLdFkmgM6hocmy6RZRVk+T0Ob2m0KlaVPgCdoTb1kHsTwBUyQ1RX1xzbt8wPdvkiijpSzHxLQv8ATM2QM+uobcPtPiRQjCLulJLcomyiS8ZLM9Zd1+qiui3RDitdsrj6nLTe1JxlNWJPoy65phRhiL5IK9QXVBsdZN0cn2BO2ibPEwGpCxLiemtQmSUfLYHD+drmNdzx9ZH9pEkWChOhr0E8fBMKiNCzfI+2LJcJpqJ9j4MbCD+aAsBmUUF3DGf+UygTKKkK3Y2RK5GKV24IjrqYP9l+Hymmr9XgoLdZ9Qh0MkcLNS/yVND8w3yCTunSIZk3jtTcRtrI2OuXUbbWuonvxxhfzM7Tcjhi11tvFrEswSYbr3XdUsE0jmo7MO+PFFFh/riKx7yTAdKzFoBJgDS0nna01Yuuhpuhb2/Q0d/6pxJEpYk7Z9bEY+R5qpnd+4lKYF/ve3+csabiRvzIEk2uLcifrflRTBJUhG6BjX7pmpv8lhP5LHUSRhaf6WMlx7gzRteehe9Hm40S7KpqK07U2fUfRixtpKocQq6YR8XrEKb4bR+0GQ536Te00aBvD8uTSAN+og+3e+QkoWM/pwavPevQOYaIqhwouweYU5kOFwrScj8jnM6ve8OYQbqV0Q0coCLUWJ6rY1f81nf20/T9+8ZP9+3WKaPdH4tRwse+RImtUj2t5IEz1XZqOnaCBJdUkbA4Cup8oxVP9KhsvPL11N4pc3CBxvilJiG0vmAPfuIwcmOjZSub9bCPz3/1E03U+upySoDx1O62n4Evhzlp1TT0jFaFp0NmsXQ7lltGX/9L7NwHAiHqIo7USY2rBj76JjZ7W0urGGhNjv36T3BdV0bEVfS1HomTzWmuslP/lcYIbD4KjLDVxuiXod8SMSZvOxw372QaB0ks5DtJWmR2dmR5rXlOij9Ae3jun/wS9x7Td+bf9cPHMoKvUrf3IdvVfddC2uXwLcZF5bol9x5kvA/1tiPiPn6bPE2nOjFgWqWz73dTb0LceJeEYcCJJhsswLbaT7ohCqBQWV5ixPc+aJsCoCgIPNfsoBF5Hf8WF5joETngDNAi0fDQyZSPfvXY62Hx/rgMOmT4HVZr20znxoWg7MO1++n3858MZNPS/HliAecr5v7jyVZmUd+Rd6FhRC7Q6T6NXlptwGk8+nyOH7DySbmHRC6hWt3wHz/08+w92xW9f36LpluQM5/zhi8/aPycuAi7JPPfUUZs2ahauvvhrLli3DhAkTMG3aNDQ2Oo92vf/++zjllFNw9tln45NPPsH06dMxffp0fPnll5llbr75Ztx9992YM2cOlixZgoKCAkybNg3xuOzAnXbaafjqq6+wYMECvPTSS3j77bdxzjnnDPTuDm6yvRWtyRx6tR6H6TF9RST6AqiTIcQAp5ud2wuc+Fjfs1KLRqqnkbJi6mmsjR6izniB/JuEKJyOU+fUV0yNbDpBnR7hI+srsk9NzYhOpkCXLcrG2mXj+PtP5Oe+QlOUNe0LALtlRNVYYP1HuVOZFIU6WxnbAT+NalpH1z1BZMQpgG4CVabnXuUuVKaCSrsoK/Y7Gy2VK9YqLnkuoy0kavhLnW0OIl2MkGtp2m5X3lvCvkA88CTCpgiWJfZ9+5o9ejcVs0TKWhCC15fPUQKieAfg8jjYF4RlPRK2G4dfRUJbtJWOt5hWpiXzJ8YTUaeijmopmmbWE1xuM9GXJXLX6hmdsS+ImQ/0bZZI2QQ9YIrIeUOXvrPdYbMvEFPnFNmxE5GV9MYUBlXYBocMzZy+7/BwqqWAO8bn375hyPWL6+z+A+3LOGV5Vl2mJ6qIjM2yL9A1EgOFUG5LluWSy3gLZPSfqpoWA36LaGnYZxJ4C6SYalhE7OwIwoynrOmx6faZHrIean9UtynGmjYVijmwJdYjyiw8ZYU1gTU61e0nAUi0D0Jwtfrouv30p6foMyE2i/OaGaAxgGyf3m5RoFgHn1pW0cPo9L/JOhtro2lqmz6Tyzm13+vep+QTlWNI4Fz6iIxqAYC/TSEh1jDooRWgDtfGpcDxD8rorEA5fS/aV7eXptpZqdkdOPZumqanummAzFpP/aVUhsoxFvsC03rC5ZFRUWLfX79WRtxFm8k2xu2njrmoE+kkMOfH9v3X9Vz/ZrEtQNbfntwvrZ3gaLOZjFKn6d6xNronFNUCU6+hZYIVJI49cZL8nTWZlq+oZ9ZIJcMp2cyqBQAUKnM6DgzdW9aNRKdFlM2KlC0bCZz8b2B2m7yfCo9qbwFFKpcOp2Ny+Gz7804yIu8XDx5GIoRVqMymff3AT/OLtuRGCv7dIsZ2NtB5MXRqV857J3OcdV+xmYCunDps08yOqIiu2W8mJfcB6Jha9zUZtUccimh+QSoij6+4v363iK6B52fm7ke4Uba5L/0BeOJkmQsgVJd7b+nYAHz+JK3vQHOwbrdf0Ku3kKyExv0UePxEErGEeOIvBa4whZW9z7KU3yJY/HkoiX7+UmDXY+XnWip3Zo+Vf58iI7cAYOXrdD9/4fc0eD10IiWqsyKOqdUKYkvyQpb/9tJHgW//R88wnZYyPXWaFI7b1pA/5o/MAQnrvahqF4oQXHQDLSMGn62MPRLY88TcsugaCYolOwB/kP07BMvl4HWFKX6uep1eXR46hgtm03V2x3h6VtvtFyS6u7x03Dd9Rtf50bcBqxbSTIXPnuzdsYq20rTonM+b6RlP0BPrnJ5S9wlFac//E72fdBZw6OXUZ1jygDy27evofjjmCHr+zsGge1c2ipL/+dLJQzYfviIpCOk6zTT46CFgyf30WbmInlRowMs6K7A/6G9/2kQntWfC5kSQmckSo/vKTocAVzRQMsrXr5Yz+nQd+PgR+l8ExPiLnQNJQpuA10wP7O8W5fY/oi3Unm74WCYk7i0PT6O6vbks+6d9dtmmz+h5pGK0bMuyB1YPvYKitouH0ADKToc6r9vt7T6YxIljbgMOMeuHnqZnO4DaeWFjBNBMxC+fpXP3gbUem3Xoi2fpvH1wH0Xb9qe36tK5dM8yDODfp1osBHvILaMpaMAaQDL2SOBXZgDA+OOojQbIUisbX6G9XTI04O1b5MBzpNmeBDF75tOHD9LAl3jus1pL6Bo9GzMDwoCLsrfffjtmzpyJGTNmYPz48ZgzZw6CwSAefvhhx+XvuusuHHnkkbjkkkswbtw4XH/99dh7771x773UgTIMA3feeSeuvPJKHHfccdhzzz3x2GOPoa6uDvPmzQMAfPPNN5g/fz4eeughTJ48GQcccADuuecePPnkk6irc/DAYSxYOs9a0h451VOs3pCbixC3ACkMAXSz2v/31BnTLZ6a434mO6ot3wH/Or7n20rHaH8TYfptd0RbaeqGsC/wFckp8gB1lEVCr2TYFLwqgQ0f0sOl0zQigG4s46fL7JWCVIRE13E/s08ZEUJbpElGA1lF2UA5ZWMWwqqVjvX2Tnn5aLt9gXgA95eSsNfwFUV4XdFAEVvxdmlfIDplYoq29Sb33SLqWGeL/PEO6jgno/I7VbVHLoobg8gy64R4cIn1QJRNhCiCMRm2X99CNHjiRLtJfyoqPWWtiOP+rBlREG4AXF6o6ZhMiuQJmPsm7AtiUrB2+6i8InJSiDHW429FVWVdMAwSwnpaP8WUcyHOZqa3u2lk3+WWQp7wCs54yqaloO/ymqJsEV3zz/3GIbGe5bxbrQGEWNe5ST7QZU9R1/Vc+wtdo46EkyVG9sNENtdXys6l9fyFm7pODCeulVTUvJZ1c4BIiLJpEgiF6CwerEU0sKLQ+fEW2BMOpExPYZF4a+VrdkHRUyAjXnVdXjfimpkxH4CBjKdsxr7AT+u2JvgSlgmAJVI2S5RVXFROT4CuTevMCLffnAIpRFlTSLbaF7j9ZuRq2mJtoMvznjlufXzgtbZN0RYS+6zJ5lIxGkiyCkZOke/rlpAPXslwavOE6PNvU4QSETKRZurM7vUrStaT6KQ2VbQNviKzrSvNX2Z/CU39LB5KdcST1dkNlFF5C6rkNSlEWYAE3VEHyWvKmkgpas5McPspei1k7rdVxBLH5p1bgSdPs3xubiveQfsljtOiG0mk64p//UL+H2mWkRdaUgooAHCAKWIFyinK2BrhlQzTtW+1L+iOP3wphZk9T6LfJqPAr1+TkVzxkLyXWkVZXzFw9uvArkfbBz6EbYungO71IqKpZBjVr0OvBEbsT+Kgnpadm9VvdV3uzrqeiwXt6/o2rTrSTOV66Cf0XksDDWbE2lE3U2fpltF2QcZ8HjM8hbTfVbvQrIEd9qHvrZFMu/+SRM9ECHhpFnX4UnG6v1pFWutgS9EQElnF/V9cV//8Of0vIvqs3DpGJij67EmyqBCd63hH7sCKGJBt/ErWpxMeAXY8QNZFl5eu8W/nS5HeV0TXpjtA1iOCG2spslyw4GrgqL/S1Ozynege+OZNFDEt6n7jN8Abf5a/WfGK/Zg8fjzw1s3Asrm0P8GK3BkY4QagZMTAJELtCcvmyrbCMIAXL6TkbECuaHbXBLrW7ppA7aaqApd8T9ecONeFNXSO376FojC/nU++xACJT43f5Lb9Z75Iz4/CY754SG45A2UkaJfuCBxyGV1vWtpukyKive/Zm67BgipqR0dMoc+nXkNBEOs+oIg6ISL2FDF4lV1+c4aBkuige3dfPWvrPpFJ0gTifp2KyeAWbyHw/t0UgZropGOejJDQs+dJdl9uweHX0ODgQOEvlmWPt1N5vniWxK0f/VY+y+5yFA12+Yv711O2v32CY200OBdtoft+62qypPjmBeC4v9Ez0ncLaVmPX9ah1y6j1+vK5L1YDHKVDAdazGPUuJySKsU7yJtZzAT858/pGVWIt6kYDTwBFM2Y7JQzWnpKKk791/443q9ear+XNH5D6y0aQoMdZSNzf1NYTW2FCO4RfvAn/nPzywPY+0dCHLy63S7IAsDxD9GysTa6Rt+5zSy/2c49dzadN0F/XlMv/p4iUB+eBqx4WSbt6wrDoIGXf/2SnmE+uI/uP3fukestLPyNZy4Cfucw+ALYA5tOmEuvh5qDAbE2+hMzQ545k56F37mdrB1euZiemTvWkxWK9d4gnnG+fa176z+m1wyoKJtMJrF06VJMnSojSVRVxdSpU7F48WLH3yxevNi2PABMmzYts/zq1atRX19vW6akpASTJ0/OLLN48WKUlpZin332ySwzdepUqKqKJUucFf5EIoFQKGT7AwBd138Qf4b54GEYBgwo0LU0DC0FXXX3fn2mgNAvZbNE3Rr+IuipBJU3nYR++DUwdtwfejICw+2j5Q0DBgz6v2kFsOr1nm8r1gGjeCj0lu+Ae/bufvlkFEbREOjRFhiJTuieYOZY6rpO5TAM6O4g9EQYeioKwxQeddUL3VNoO04AYLj9MPzF0H/5CHSX17Y9ANBrdod+wlxar/jO5YOeisHQNXnsvYW286GHm6B7CnP2AQCMVByGocMAoJePgu4JyGXM60L3FsJweWFEmqF7CqhsigtIRWEEK2G0r4MeKIfhCcCItgDNK2A8faZcT8M35rZi9uuusx5G9W7QI83QzamFBlTal8xxpg6aboqyuq5DT6fs+xLvNJdpyX+Nm9e3Hm2DUVhN50xRM+cLfxlBywXKoacT8nepKHRfCYx0HHR1gc6tywvdIgoa0WYYCj1UG6qL1uHy0e9VD/0mFYMurlXVAyPaStfNPmdDP+tV2geXz7n8iguGnobh8kJPxal+Ki75vZ6GoajO+2+Kt7ripnOguGBoSRiqB/qOP6ZloMJIx6AHK2GEG2gfoMIwNOgu87pSPdRGeAugJzphrH4HetsaOi6GAT2dovZDbBcqdM08V1oKuqLQtCJz6pbosmbOj56CoagwTGGJ9kuD4fJBTycd62BXbQ30NHQzKkm3REfpXzwL47HjMvU002ZYrxUYZtvih66lYKhu6OK61FJ07UCFrqXltauloBskS+upBIzaPWBUjpH7ko5Bd/thKAoMIw0jy1fLcPthpGJU18xzDUC2G8Mnw4h3QE8naBnNPN4uH4x0nM5vMgpDJQFZNwer6DrXYZjvDT1Nx1V1UVvvCcBIRul8ZdoVElh18yFYhwLD0GCobvo9DBhuHwxvEIaWhJ5OwnB5aL1ie7oGQ/XAMMWJ3rT9BgwoppCo6zr0aAt0fxndk0T9TEXpek2E5W8T1F4Ycw7MtBNGuAF6QQ304h2gt66BUfcpjImnAStegR4Pwdjt59APvQJ6w9fAgtnQh06CEW6iY+0rgRFtoWveMKjO+kqcrzdQRGLms+KhMIpq7cv5S2C0r4fuDsBoXwtcUwI9HoLuNttdbyEMbyG16e4AjHQsU6+NSBN0fzl0s/OuJyKZe1fm2k7GoG/6AkbbWhidmyx1hdoq440bYcw9DsaIKfT33SIYXzzb9blY+768RsON1IaYbZ+eCNM9xnpPCZZTe2k9FokwDF8hHVPzXtnTPwAwiofBmH8ZDC1B14Bot9Mxeb9T3XQtJjqhn/oM9GBF7r7sczb0IROhewIwUhHopsin6zr0oXtBP/CP0EcfCuPhaUCkEUbHRhhmtI/tvpT917ERRtTh/rP+YxiP/Vy+1zQY//0djPmXyc/WfpB/vR8+CD3eCb19A4yNS2k64YYPaV+eO1ue90A52R/FWukazT5+igpt6rXQj/gzjKpdodfuCcMdgB6wHKNAGfRdfwajdTWM4iEwdjoYeqwdeiICw9Jp1bUUDHPAyNj1GOjRVhjmQLBhGXjVvYVApMm5npjHMtMmmfcJI95O58+6/IOHyXVarjX9zBehFw012wISg43J58t2z7wujMJq6N4iGKteh750LqAloXfWZ8qCjvVUb0cfDv2CpSSavnMbraN9Pa1/9dvAW3+Fvnax3IdYh22fsOTvMEYfDqN0hHwW1TT6e+d26A1fwxh5AIxFN8B4aCr99p07oDeu6FV96NOfpmX2x/j7j2GYfoRG03IY1eMz+2I8S9eUMfxH0NvIcssY8SNahycIY/lLMMp2guErouvn9BdovTV7UFsfD9H98PZdgb/9CFj5Wlb7WALd5YVRNhL6yAOgH3xZbln3+TU9E/lLoR90KYzS4dA3fQ7DFFaN8p1gbPgIhhnZbGz4iO45epqeif5vPfS9Tqf7XqwNRuVYGJ88Dv2L53p+vEx7Jr3ha+ht6+Tn4QYY3kL4v30Ryj+mQjdzRTg+nzi1pf85D/qqN2A8+2sYHz1s/94U54yNy6APn5w55pm2953bYBQPgx7vhNG2BnrtBBjBytztBCugn/vOwF1L7gDQsR6GvxR621oY1eNgmFGlhmG5z9fuCX3CKdA9hXRd9HV7DV/DuH2cfJZzWlcqDj37ePb0L9oKo2Q4WdTcPZH+1rxD15OYPQDLfu07k57ViobA2PEAGGWjoP9pLfTffSKfDXY+Aph3Pt2PW1YBS+bQsdqJ2jJ9+hy53mgrtRFrP5Dn2gwM0q3PNt1dW/fuB9xYA8NfAiPWvtnn2dDT0F0e+d4U4vXCWmD1W1TH8v1W7FvlWPps15/mX9YwelamGa9Br91Dvjc9anWn3wfK6Xkp0kJtwMLroa9+G7o5e8EYti8Mi76ghzb1rAzd/WmWWRimaKz34FwY835LAy+rFshrIN5BA7j//S2dC+tvLquDXjvRfj+0/EFPw1Bc0M99F7o5UKV7gjBKdwTWvgc90gxDJBNf+T9gxSswVi4A5p1PbbuehvHUr6CPPgxYNhd6Rx1wTQmM+ybTb544EXo3z438ZzkfPaQPhqE9p7m5GZqmoaamxvZ5TU0Nli9f7vib+vp6x+Xr6+sz34vPulqmurra9r3b7UZ5eXlmmWxuuukmXHvttTmfNzU12WwRtkd0XUdHRwcq0mmkYzEomoKOuvUo7uxAuK0Deqp3xuruSAqVQF6Lit5QUP8dCryFUJNhpNQg2hvqoAfiKE0m0d7UhNJEAqG6tSjUXAiZ2ytNJNDe2IhA/fco6UU5PPXrUFgwDPrqxQj04He+lgb4vWVINW2Au6MZ4VACut5IZWtsRGk8jvbGRrjCCQTbG5FyVwI7HIbSFS+jNZKEGm1BuWU75TV7wRVpQEz3Iuy07ZlfAnEPELd/5wnH4G1rgieZRHtTM6p9pWhskR2oQCwFd8s6aMUjEM1ab1VBDeKJNNyxCAxvAUJDDgPcPuiW5WoBNIdTKEwZULxl6Mj6rtNdhuJUBC1xBUHNBVeoDj4A+voP0WQuG4xEUQwgkaBjIigJt0H3lcH32HR07nsRygAk02kYsWhmOTXWgmoAseb1KADQWF+H2gd2Q/1vPstEELra16E8WIXO+jWIFzXCu/EDBL96Au1H3J3ZVmlKA7QoYs11KPCWQAu1IB6OIpBMoL2hAbUAGjdtQKWvFPHW+sw5KA61IhLTUdjZBkV4ZwGIhOPwtDej0F8KNd4OJd6BjkgM5QASKR3JVoo2LUqEEY4m4E0mkWhtgh4oR6KxEa6OCEo7mxBLGoh2xDPHs70ziqTD+S+MJRBIp5BMGwht2oDiSCdCre0wzIH0sngMutdlOz8CXySOMgCd4ShKtCQamttQkYgjmUij01ze0xFGeTKK9riBwnArUsEdoEWiKNLTaGxuQS2A1lAY5aoHibSBdFsDAqoH8Y4WuBM0Vb5j4xoUa0qmDMFoHKnWFqT15XC3t8AbicEaa9bZ3papo2XJJJDWoCsB+M0owcbGRpREwzBcXkSa6qGl7KPhaudGVCN/Xa0FEGrcgFIAoZZGlJqfh8V+NTaiNEHCn/W6FJ+l21sQUL1obWpEia4g0RlCpLERgVA7dM0PTzyOWHMTDG8c1QBgaGhsakJpSkO0uR7ust2hFQ5BmVnGmlQM4XgarlgC3lQKnVWTUY6HMttNqgFonW1wGyqQJC9YL4CkDvjE8fCWI964Ad6kDjUeguEtRDqeQlEqhvbOKApjISTiKRQBaO/oRDmAUGcEwVQKrlQcLgDRSCfUVAQuzUCyswM+xYNUqBWR9k5oqnkctCRqAXTENJQBaG3vQGkqiURShxqNQNES0JI6vHAj0dmBSMMmlGqAFo1CTaaBdBSG7oJukB9wQRfnyYnSeAJ+U8xtbGxEQdN6JAI7QU1F4G5vRbSxEYHWRui+EgSiIcQ++BcSOx0Bb9MmlANQ6j9H86ql8G76CIUrXkPTpEuguIeg5n+/QHzHQ9E+7XqUNa+BNu8P8K95Ay17no/C926HsuOh6Kg5CMUrFkJJRdDeFkbt+g/QWfM7RBobUZ1OIBxL5LSlAIBzvgLibtlGD/8pMOxIwLKsN66gsPl7RIcdjOK2dVAAxOq+QbxwLFLmciWGD5F136DMWwg10YF0Oo2WxkbUfvgAGnc/F4oWRTWAtk2rkUIF3M1rISaxtjTWoeqpo6F7i6AV1KJFtKPhelQEKoDWNTDcQaTC7dCLd4avbT3cX89DQ8Nf8s5yqXZ5MwJ5sr0Oiq6jo3ETqgB0NNdDTaYRM7dTCyChq3DBg1Rna+a+XNjeDJ+vDHrbBrS1RwClmyh3C4UTZ0JzV6CkeQXQvAINWfegjgSQaGyEGoug+sMHENnrHMQrEkg7naOJvwMABOIfoDgZQXssbbsPA0BAD6DQAKL7zULgm6cRHzsdBa73kIhFbO2ErYwN38GvaWjO+j6w4i0Ur34rs/6S1/+IwOo3ERt9dKadLJ9/JVqP+5fjemtfuRh45WJEd/0lgsvnwVBc0Apr0f7N2wgaXsSPnYvk0MnwblwMs4sFxdBsx8g4cwna29thlOwAVVWB4/8LtLSjomgHtLTarYIUrRDF9d9CKRgGTXMjWr8enuZNKE3JiK3O9hb40gb8AKKJNLR1n0PVfSgEkPaVwhOm5+xoawMKFFdOva8F0N5Yh6SvEVUuH1wIIx6Lo6OxEdWxdij3H4jmE19E1ZNHof7c5agsHQV3+2oAQEtnHJqSew48niHw7vcHRPY6D2hspPt5M82cKaneGx1NzajVU1Be/D0AQH38eLQe+1jmmLUm3ZnrpXTEIfCvexPJmr0QXfEu4qhEsKMNxQDiHzyC9LfvoBhAqGkD4mWNgJZEjeKCYmgIDT0QsZ/8DTCfAT137IHw3uei5G3qX4T2vxzFLU9AdwfRWL8JNQuvRWzjlwgdcqPj+e8PlHgbShb9CX4A6t+pg56q2BXhafeh6P2bkK4ch2jDeqSUClR/PQ+x0UdBScehffQ4CgB0phRq7wwDtfF2oI6is9pKd0OiYBeUjjwcYc8QpH/zKUpfuwCd338G6zyvRkvb0BJOQmtsBI57iqJBYwoQyzqfZfuIHwIACtQSBJ88FWqYRLlY9d5wr1mMtmMeQrEGJIYfCE/jF3DBZ6mfdL1W+ksRS7tQFK6H8tyv0ZpQkNxhf7mtdJzK4fKgcMkdiO0yHVrpKHjr16Fg+EFIffgYCpfNQfOJLyJdPhZFdcvhLahFJKmjRE8j+b9r4APQVLcORnZirWwMHbWf/RvKZzS7qFNzIWKpG/7mOrj2+wOKPrwDbf7hSDY2QklFUQOg9di5KH/xTMRGHwW9ZQNciTjak354djklc9/YktQCSJWORvS7JXCrRQiGm6AVDUNH7QE55VFiCZQ0fY/2jWvhCtdBKxvtvFIntCRqH6RrtuPzV1AKoL1xA5Ieu0+wGt6E6pf/gPrhR/d6X0pf/zN0bxmy56ilSkahtbERJWN+ho5D/2K5jxejePTRgOKC/7uXoaaiaOiIAwgCTTJ6vnj8KQivWwF3OI5yAOr9ByI05U9A9T6IF+0GoVK0bliJ8vsPhppoR3jv8+D/bj50fym8sVa0NmxAWut+Zokaa0G1GZGZ8pVDa1rj2BfoDTWGjkjTehgLb0N0j9NRUvc1AgBa9EJUAWhUqmzPNlZcI46Cq3xvJJu6mOEIqT8YhkH3pq7wjQRaQwDMKOAdfwbfET4knMpgGCgNNSOyaTXKY21IV+yC1NqlCH79FDoOug7JYT9C1b+PQLJqD3ibvoAx91g0nfHeZs30LXn9jwgdeA2s6lR8xMFINq6l9jOdgKdlBVI1e+b8trz+G2SbRcZjUYgeT7h0nK2tIPJbgpQN2x+Rvc5BUqkCwjrdD9ujUKY/jZpHJ0N9/HikS3fKiIDxnabBU78MLgC6pxDK+iVQ4+1oCYxCJQD1jnEAACXWCsO816n//S0ai8bD07ICiR0P6fb4VP57GkIHXUv+9j8gOjt7bt0yoKLsYOKyyy7DrFmzMu9DoRCGDx+OqqoqFBfnmWa+naDrOhRFgcfjhcfvB5QAfKUFUHwe+KuH2Ke39YSqSugXLEN1eS9/58TEnwNjDgT+NR2eokpU1r8F5av/AMFyVFdXQ/H5UFkShFJSCb8pxCs+H4nya9ww3H5UV1bkGt470aZCqR1LWWSB7n9X74VSORI+dxqKW4d/6AjAX0LbL/JCCRZROQIalJU6jKDXNEAHymtHAKUHQh93CKrFVJ9zF0G5ZTSCZbUIVvfi2KWHQmn8ACgspWNSXGsflGiogtL0MYyKISjMXu/57yPg9kJ5/JdAYRkqR0903ETlsJ2gFJYA7hr4stZROHwP2qfhY6GsrwQ6voV+6Rqo/zkX1c2LoXw7H0bNbtAPvQK+DR/byqa4VRi1Y6F++S+UBKk58vr8gMuD6oAGfPZvYDeyoAgaERjeQlSX0YNvdaEbKDbP+cd/hRJtQrEnjeLqamBDO9TvX7Nvy+cDFBXeoAdKcTU8Rgq+sgooXh+qK+kcVJcVQSkZggK3ljkHituAf9hoKMs0IFBIUzYUBd7KWqDdDWX4ZOh7nwn1qVNRUkHTdX2BAviCXqCgEko6hsLScijNQXh9ClBRA1RXA/4UlGQIheU1tvNSOnQn+j4LpbAYUAz4C0vhKy+G4nXBVzMkM6VW8biAQDDn/AAA2kiyKSoz97N2CBRFh7uoBAGxvFYFRU+htGYEFD0OT0ERjOISwNAzx7G8eigUtxf+YBHg0gBfAQq8KuDzAe4AqkoCUApLZRmKS4CiINS5U6Cf8RIQKYb+6wVQH6bpt0WFARiq26zLfpqmGCjITN2mzz1AoBT+0qLc46K2Z5bLR7GPRPTiAumFV2jZL8XrBRQl91oxDPh8LsBXgIqKcii+ADzBAAqqq4G1AaCsHEpnMYKlxYAnCMNNFgvV1dVQgoXwFvoAlALF5XJf0nEUlldD0doAVUFpWXlmm/pvl8D73l3USfSblidmlIw3UCTXUVQOn1+FUlQC6J1AsABGaSWdu4oqKIoGdwlNyyqtoK5xcWk5FLc74/8V9HsBJQH4AvAEfIC/CG4P4K+qBirM42AOPpRUj6BzX1EFRQWChSVAup3CnIsrgLZiuP1eBCvKoAQKAL8X0E27Ba8HKCwGEh3dnqccfPJ8VVdVQVHiKNiBfFiVta+isLwY8LuB8looPh/8//sd9CubgDbZZlcUuKA0L4MSqUf1EJpWaIyfDl+0FdW1Q6EM2xPY8BGMA2ahYsy+UJYCcPtQNWwUFCUFeL2oNgeAC4aNp3P/8/tRWDI8ty3tKdpIKNF6FFcNgzLyAGDlawgihkD10Mz1rRSXw+cHlGA5lGgT3B5f5thVDRtF0xoBlIW/BVxjoT57HO1b9XhUlBTCCJRBjbVBqRyb+Z3y6tmATlP7FaMJrppdYVSMh1JH10R1aUBO+85CKR9FU94B+DZ+AGPoJFSUUDtcEvQARTUoMrdjTD4fvrY1gF4It1uh+/LnT0LxAijdAYi1oLqm1nE7efnZzeSv9iagz3gt5zoqGTaWjp1G131BvB7BITsCpV2co7axUAwdpaP2zrRB8rudoJQMRcHUS6F8dBeCQ3aGcUwDfP8+yd5O/HVHGJeuARQFSrodcLtR88LJMH6zSK6rsACwrF8JUQSU3+fLtJNKokWuNxmRyfssBErM43tFPVw3VKPymeNg7P5L+McfQufNsAsd1nLqeiXQ1ISqqip7x/fH/y+3ThpVUIwYtcMlFQh+ei/55VooUmNQgtQmBQtLoLx/E4zJ5wEA3MW1QAsFXRS4yWal5r8nwjjteVsW+dKAClRXQ/EVALEW+D0KfJUVUM2pwZUf3Ur7UeiCUr0LYIqyFTuOz52qCgDVRwETjoIwC9H/bwOqhfXKyQ+jGoD+x5VQbxuT+Un5i2fQLo86GOW7W2bonf40cGM1PL4AivV2eqYopP0NhNcAfh+MCaeiuMBP37WtIVuIxq9RdOiFmYFHpbAUyrqNKK6XkeaFoyYB7wNqOoraB8bTMVz+LPy/nNOz59S+8P03UNe9BUN1QzEtkNzDJqJkxz2gvrYB6uSZ8OrNgDcJRU/DHwgCCR1INkH/fx+jsKgWhVmeo/qla1DiKyYR44xn5YCA1wNviryy9d9/BrStsV1jFaP27JmftJUhY6F+tIkiZP/fR/C/eweUtYtQNXwMcMpj8AF030xG5fO04Ny3UOAtgJFqhrJsLkrj6+3PEe/dSZYAE06BUvcuCsLfwzjlKWDld8B+Z8H3HNmkVD59LPSf3gkl0QijcjQKvCoMlxe+jYth7HEiqsqK8/i7Anjrr5SIrlJee8YeJ6LQ76F7CgDlX8cD/mIYu/0c+BAoHb0PUFKduQ+X7vUzGJ9Ohm/kflDWfUD3piHDAPO+tjXwDN0dpYv+BP2IG6F+FgUOvx1lex6Vu2C6BOr3r6HGHwTC9TDOeiX/SqOtUF6/GsbP7gEMA+r1dD4NxYVir3ksgu7cZ0GDBKtePWOYqKv/B/3Ye4AVz8vVVY6Fp3wEre+UuchZ6y/nkL1N/W+AR6Y5blep2AH+AICU9PEtLC4DJp+LQgD6H7+FettYlAfdUBPtAIBgWS1w4SdwmdZ75YVeaidfugjGT+/MvxMtZpT1kInwKCo8SND9RUtBefgnMGa+2fMDYhjAO7dC0dMobP4Uypp3UJRuAZI0db1i50nQL9+EaqfcIYIengehP+Tcm3pK9el5v1LXvgFfvBGKloS7sBLuBAnERfueDBRUQf/J9fCsXwI0fQFXrAXVBei93iEwDKirXoJ/7Rv0tmpXKE3L4T36JvgWXY/CijKgZRWU75+HsYd9Nrjy/Ewgbs4EPfKvUOf/CfqFnyNwF4m3+vH/QMH46ShQenF8zn45E4QCAPqv/4fqocNtfurqvjNgNC2H8unj8H9PdjD6Jd9DiXdAvYeSfpaP2BVGYS0Uc6DVGHsUjMnnQ/nnzwAAlaEvob56AfQLvyC7qmxLOgtqxxqUKp09vja2F/z+LupJFgMqylZWVsLlcqGhwe6f1NDQgNpa54fy2traLpcXrw0NDRgyZIhtmYkTJ2aWyR6ZT6fTaG1tzbtdn88Hn6UTKFBVtW8NxSBDURTqWACAJwhFiwN6Corba/dj6xEqUNmLkdCuGEKCH2a+AeXzp6EkO4GNHwFjj4KiqgAUKGayKkWUU1Hpfy0BBCugpGP5/VutJMPkH/jNi+QNmop07R2oJYHiIVCWv0zb8RVljpUSWg9Uj6Ny+IuBZJQyiXtJZFG9BYDLBRRW5qxT8fh7d8y9QUp2ULETba94qDwWAHlCRpqh+Itz1yu2n4wCvkL77yyo3iB1hIpqc5ZRK3ai15JhJBBGW6GYXjVKIkQd8kMup8yRG5faf68loYyZCrw+G2o8ROfOvPEoLSuBRdcDux5DxzdGfoqK6WOjpqJyfzxBYK/T5WfmQ7+t7oprPB0ncbx9LRQXCXKKTtPPVT1N3mSJTlnOVAxKoIx89bxV5E9UUAXFGwS0OP3evJEr5vWiuNy0HU8A0NNQ3D6qV/F2Ol+q2emOtULxFcr9uOhLqMXDnM+/y0MPgR4/JUAS6xXLmj5/jufQTO6gWo+LlgLcPrm8+M5XkEkGprgouZhYRvUVkS+p20eeR5W7kDec6gZcbqqLHr9cp+rO+P2q375C9atmvLk9H1Qz6Y6iqtQhTUZIODQ9S1XhL+wJQDG03ONi2hx01UarpreWavEnVU3PU0U1/V+hZB03ulagJWjbMDIeioqq0kOry0Nlh0GiY6Ac6Kwz98VD58jlzbkWVeEhbGh0fEWZXB5aNhGiY5DQ6DVYAcWMGFVVldq6ZKeZ/CwFqC66FgGoHvKWVUzvWNUUYVWXm/IH7nQIcNDFUN6+la4XkejL7QfSCcf2Xi0ZKteh6+b50ekYefzUvuhpKmMm0YsZbWDodK1E087nafkr1C5kJ4t4/EQYnXWyDDCAWBuUggrytPr8KSjjj6Pz4w1mtqe2raEECJ4gkIrSuTeT8WS2XbIDsHEZnafqXYGlj0I5/XnLfitQXGYCQk+Alpu1HKrwPtztOGwWwXIg0kRt5MlPAAuvgVL/pb0d8AahPH48ZRkGoKguus52OQaqy5UR69XW76Tn6dG3Qom1Q9FTtI+xNiiBUnldr33X9LkkQVdZ+RoUkRwHZnsaa6PzJTzLAEr2UbIDJY16725gwVVQtARtByAP7UCZLPtRfwEeP8H0KY7T9uedTwm7imqB9jV57zNdMmwicMl3ULOFj2Al1Iqdafuqz9y3/0H5xYNd30dHHQD87B6olaOBK5vsZSqqBQproXgDVE/9JfLeHmkCimooeWYiBCUdpftevIPqUd0n9nXN/xPgtrSJ5rxOBbpsSzrr5fd/MSPArm4Hri3NrEYx0kDlWKrTR98CvHsHlGSEyqYosGW+P++9nLqmKEru8+w+M5yPTaSJPGe9BeSvnklCSqgfPwyMPJDW6/bRsTr8amDJHKqjIG99JdFJbdqmz6BEGuh5w/SYVRMh03vQHFRMxahdE+U1PRzVSAN52V/4GVA8LHMP6xa/Q3RZkaUzOPJASpY09RooE04WLZa5gz5g1jdQQnVQHjqc2ok0RV6Key92PhxKrI32obOOspArLvvx/fn9wNfzoKx+W67afF4CQAlbvv4vULkL1BsqKXO2E7E2wFfS9fXcFaLstXuQl2n5TlAKKqEU0YCTOnwy8PbNGa9YJRmV98WqMbnrO/hPUINluZ8DQOM3UJ45E/jpHVDLRwLlI+V3l2+C6sntZ3WL6U2t/P4TOk+FVfTsZHvODWaer22IevGzu4FDLoP65k10HJtWkBhb/wVQ3EKflQxH5h4fbQHGHQv84kFgzTtAw9dQV79tPodpJKL5ioERP6L2ec3b5P167tu5ZVg2l+6FRTKGTimqAZJhuQ/f00COsh8lxlNLdpDn+7INdF0FyqEEKyCSn/apHe0vCmuhmL616rC96bWo1vka9dIgihLaCITq7OU2DLrWEmHymf7ndFp2+n3AA4fQModfDWXInlDev4e2k4rlbse0V1FjrfnF8WzCTUD7WmDskVAnnUH1+L27gCP/AiVYDjSvzH+MVR8AH/VR9/qV8zNoYRWUeKst94FaWCXLXlQDHHM7VEu7p4rnXPHc+P0bNOCzbC6Un93tXJalj5I/9OTzoXgLgM+fAgqrqewta6n9nftTYEYXYriV5lXAm38GACjtZGGiLH2EBj5GHUTPmf2I472pv9bd8BUw6mAo+/8e+PB+OteiHv7495TIbOSBwPw/QU10Ontc9wTTd1VJRYDDroLy44uA0AaongLg21ehfPE08MIFwM5T6bwko/SspbooIZmJutevqCylI4CdfwI0r4A67lh7voe+MMK0HVBV4Nx3gNcuh7rzVKB2d+pjm8lk1YIK8l+/pgOItUMNlAIXrwBu3w0IbYBi6FAs91F13WLAVwL17ZvJ599sCxxxeaG2r8v0xX8o9Oa6HtAW3ev1YtKkSVi4cGHmM13XsXDhQkyZMsXxN1OmTLEtDwALFizILD9q1CjU1tbalgmFQliyZElmmSlTpqC9vR1Ll0oD5EWLFkHXdUyePLnf9m/7w0xS5Q3KbJPZiZm2FsP2po67SH4iEtYoKnWenTKHpkmUdczG7UQiRJlCoy2UYMBqlJ5OAH/b3758Kg5UjKFEAoYuG81v5wP3HySNwz1BKoNIiHLYlfnFXl2TBv89JVAGNHwpDeZ/mZVEzx2gTpZI1uREoiN/EhOf2elzecngPZvioZRwA6DzIDpbqpv22+UzBU0pnkgMStSz16/Ix80ThMzybp7jZJSE0kgLCV/CZN+a3EZ1k+CUjAC3jkUmIZYNc9upOB2zZFiO6gnDci1BokkyYiZDMv/EtScSwXkLSZAyfewyEUBC/Fc9ZqIlswPp8tLysXZ7oi8tKQ3xAUo2kPch0Eye5PbRurW0PeLE6OLaySRRsyyfXb8zSaE8MhmO6oYtcZe3ADjsKjpfAAkRWkoum+y0J7ZSXZZs3G+YEWBmx8ntN7djlklxyWR7VnSN1pmdpRZwTuqUjTC8t5rSZ+9XPtJxM+mWlumMUZnE8TETfcXb7Qk3VDf9VixjxVckE4lV7AxMMoUR1SV/5/Iik1SrbJQ94ZUnSA+AbrIFoERfZl1xeWUiN7FOQCYWUxQzOaI10ZflmnIa5RZ1XlGlAC+OnSdgDjykkEk+Zk1uI5J/aUnna/PJU0gYyWbla/YkKFqS6qu3UO5bIkz1z+OX5RHJNqp2pdfmlUDpCHuW37KR1OYANKiQjsmEB26/TPAy+jBKUAj0/UHdCZEkx1dI9wxvoZnwz3IPE3Ukcw/JSpDl9lLSnUizPN6xdvo81g4Um8KetX4XDwOm/12+3+VoeW2PP47W/8yZlKzIygsX0DoB6sSc+RKJJKI+xtpzIxdXLgBGHWy2U+ZyyShFoWxOFm6nzval39mTOB11C9Wh7jqO/hISioHcNrd4qEww5i+RCct+fCElUAMo0dOIKTLxhaKYoo4Dop368EGg6RvzMxpQgpaiay47aeGC2fQqfN/CDcAJj9L/+80Eavc023xFllkkU6ndzCQ/kSagoNpso7Pa3Yox1BaIa8vlRea5sWYPMxmf+V28Q97HI820j4+ZgxrfLaIIU4tQhc5NlDgUoHqy4wHA/Mso4UjZyN5HWDoxcxFw0KVA9ThgzxOBCSc7L1c8VD5ThTYC799DbWfRECDaTNdyOk7XeMdGYMxPgPPfta/D46fObSpK2wVICBT8+CLglKfovt8Vt+0KfP2fPu0uAJksaNRBlMjMX0rXtDjWlWPofLz1V3pv6JQI7dtXc9d1VYtMGOOESGBXOiL3OyfRtCfU7k4JlwSiTvSWwhpKSgfQM/obN1J71LravtzcY6ntDZTTNbLToSTMN35NfQOXB65oo3mv9gCr36HEQZs+k4mGBYZB9aVtDfnpA5SEa48T6R4m2NmMnvMWAhd+bm+TRLsfKKP2SHHZos63ChevoOSIALDj/pQkuGLnrn+TjlPSPmtb9+qf6Nm+8ZuMIJvB0IHpc4ADZ9G9WiQNTDrcQ6It1P40mdaIK6U3J544ybk8t+4MPHQ4Mn0DbyGdr0lnkiB/4Kyu9wegZ+Lj7nP+LlhByTlbVtGxKR8N7PFL+zJmEEjmuXnoRHqt3YO+e/sWSt4I5CadE7x4Ib2uWkDXRbRVfif6sSKhXz6s1+29k+T/7euAC5bSNTztJkrUN5j4zULgzBeAMVPlc7WVHacA+51D/2/6lL6/cWj+Y51N+zq6l8fa6DkKAHbYl57tykbK5736L+hVJIf88xDgunIaGBBc8r3spysKcOrTwP/7UD7f9xdD9gTOeokCZEYfRqLsH1fINkhg1SlE4ERnHd07Z7dRP3DjUqr/ravtiS+d8BbS4N+1pV0n7v4BM+DDbLNmzcKDDz6IuXPn4ptvvsH555+PSCSCGTOoI3rGGWfgsssuyyx/4YUXYv78+bjtttuwfPlyXHPNNfj4449xwQUXAKARlYsuugg33HADXnjhBXzxxRc444wzMHToUEyfPh0AMG7cOBx55JGYOXMmPvzwQ7z33nu44IILcPLJJ2Po0KE5ZWRMtCR14s0oI2ipbUeUBagsQmAVndZAKWWItQpbojFNxeimZhXvuiIRkp2vsh2lmATQw1PjV/blRWd+7JHO6xMdatUUGZfNpU7OQZfkb2TPfZseCHpDsIJGe0V0k7gJCDx+aiy9XfgSJTrzTl3FZevo1e23Z9MG6GbtLQBmvEzvfYXyHBVWUdbJQBk1wAVVyIuvmB7WnKYlpiL022izfMgB7OdVZARNRqjz2vp9/m2lolSmRFgK6aLTnIzSfhoGZdB87Qr5Oy1FywtxxO2n/11eehAKVkhxRxUCoylEeQvooSvWJkVZ0Vn19LCjIkRZl4/KK7LcC0TkoxMZ0S5blLW8z4iyLik6ZgtpniCw9+kURTXlApnFXYiPibBdDFFd8qFQCHOis+H2mtsx3yuqc5tj6LROkVXeSneirLdI1mPrsuI4dffglYpL8djly0TwUrldtD9ais6rNXJIddFvnY6ht8A8xjpFIB57p/kbU9gW2zR0Oj/HPwT8QvrOwhOkfXKZAwWKKh/oXV4p6gJS4M4IqYoUks0oZRENTOfRQfQQ12vmurDsjxkBDi1tOUZWUdYcrNLyCL4A1RMHFPN8GS6vFIbE4JD4PxW115/OBnrAFqLUe3cBPzqPBsIE+80ETn2S/h++H/BbS/LPgiopspXuCIyf7lzmzUF0sDMCq4/aNKtgIdpB8eCaLcoCUohMhumesu/ZdPyXv0Sf7f5LOkYiyUDtnhR5AQCTziLBoaCKMkQHK+maSnTKgaUbh8j6EbN08kYdKK8b8V1OG2aQ+ANIETYVI1Fkc0TZnjD5nPwRhz2leAhw5F/o/4tXUacDAIbtQ/cXgAZiIs2Und4w6O/Up0ns1nUS6yIt8ncADZ4KkmFaRlz/sTbLIF8FRd0BwG9eB/Y+k65tm2elKfZY6WrgtTeEG0kAz7kfKxS1FWuX9w6XR7bN55vR2OK7eIesr0+eBrx+LbDrTykC/Ot5dOxUN3CeKRj8fX8Z+R1ro22teYeiZPuLYZOAw64AjroZmJqbR8KGv4QGpUN1tC/nvQvsfDglRvEUUJv5/Exg3WI5EOLE0bfSdktG2AMIqnYBdjkSGVGoq0zrQkzsDctfAVa9Djz/G+DEx2h/f7eMIuZ+9Ftqmw+7Soqc8Q7gwD+SUB3aABziIL52F7F1opnpu3Rk78ubj6JaYK/T5PveTOO1oqoADOCRY+jcffgARWEbGvDiRSRC+0uB1W/TNS5EiZrdSWgJ1VHb4C+Bq2M91fXG5cBx95rlHAqY03wzfPEsifpfPA08fQZdC+e8RcJGIiTbWHF/9BdT/8MJIcruM6Pr621LMfZIYJY5yPT7Zd0PLtR/Tm2FNdN9w1ckWDv104qGAhPNRFulw4FfPQ8ccxtFlwtxLZ2kaP5nZ9AyyQi1q4//ku5pqRgFyiQc1j9kArDHCbI8e54InPZ0rw5Bl5SOAL54BnjrLzRI6dTv8xUBS+bQfeDYu6QwdvhVwMUrqf0RxLPua9FW4OsX6P9JZ9F15Qmas9MMuvZeuQTdoqWA6ytoeesz8anPACP2Byp3pmet/hyc3hJctoH6KgJdc247VBUYuje15deV0/GzBnE40b4OaFsLzDmAIpPb1sj7vHXw2OWmtvfD++l9OEu4vNUcyBj3M/lMJ55fxIzKLUFRLfCr5/J/f9bLwNRrqL0DzLIF5aB76/fd36OsfbiO9cDnz2xWkbdHBlyUPemkk3Drrbdi9uzZmDhxIj799FPMnz8/k6hr3bp12LRpU2b5/fffH0888QQeeOABTJgwAc8++yzmzZuH3XeXI/+XXnopfve73+Gcc87Bvvvui3A4jPnz59t8Gx5//HHsuuuuOPzww3H00UfjgAMOwAMPPDDQuzuoMX7+APCT66kRSApRth8iE/oLl0+OkArhJ1BGjaNTw5WOU+PYY1G2U/rJlI6w3wDNKdiZThNAjbYoh/VGdsSN9GoTyAxqtLprYKvG9r4RFg/JIqojGzdNk88bCQuYUTvdCFT7ng0Mz4o0r8waGfcWyijVoiHUgAdKqfNqjSTMYBmhDjeROCsabiGixUOmGNtCrxGz4bdGQGcL8PH2/AMKqVj+SFnRiRSRIpbkJtDNSNlMxJ6PIozdflrfpd9LYVt10zEV6xeRtbE2ec2IuuUU5e2EiHQ0p5rnkC2YWbGWS5AtgIrrSNgkuLKiPEtG2KM3PEHqzCQjpqDoIUHI6jOluOTDcDph/73LJ49ppkzJ3DYnEynrIMpa66PjfhfmRsq6fPLBrMs2zqBrMFBmRnx7pOAoIkZVFfjHTyhSxhq9o7pkFE32OUnFZDS4FcUSKSsiGI+4gUbbC6toGiVA4l0iJO0LbJGyHjomGVHWIrQD5rQhl6yjikLl8RU5R8qOPdIimrtyhX93VqSs9RgBJE6pnozNgiPdnUO334ziUOQ+ArS9dNx+vYXrzcEpc9nOOhIj86G6KAJK4C+WgwgTTpICQ38iIhtFnXQHqHNlFTbF/8fdI34E/Pe3Dm24QvfqslE0MOT2AW2rKcLnl/+gDnxC3McMOpcH/tFcvow6pQDdJxMhWp9h0DlJReX0+ZSDWCTqU7Qt955VPlp20B86nF61JN0LehLdvi0gzpPVWsPjB5q+Bda8S88dh/wffd76PV13qkrnKNkJvHsH8MDB1NkqrKb7iTUib807FCUmjmOsjQYexx0LjJlGQtDMRRSxW7EzRZFaz3864TDg00+ibM14egbKFtuPfwg4fDY9D1kjZa2zGKyibCIk/092Akv+Dux+PHDSv+gzTwG1KbW708AyQNfnSY/T/2J/B0KA6skUSm8QOOtFYPG9wC7HkFg2whToxeyChq+BpY/Y/EJzMKek4w9mtNQMMwJV3PtFOWKtNPPqo3/I30aac49xT3nrrySEA9TeiH32BGR7fNDFso2PtgKTzwN2/wVw0Rf2GQY9ZfxxwI4/7l6g2xyqxwEH/KFvv/12Plm5CISI2vodsOfJMnLbsNzrqsaS6CUG/30lcEU2wRh5AEU0jj6UltvpEIqattL6nYzyHX0YsO9v6NnK5aE25NpS4LnfyPtBV1HAIybTYNcuRwEHX9q3/e9PVJfd6qY7TDseWyRn5yagYx1d+9aZeOlkbv3c+XAZMbv+Q/rsowczU69ROoKez8VA/D9/TkI6IKNJrRQPAyrH0jkCqJ0edVDP96c7hu5N19vB/0disoiWt+IrJrH6V8/TNWZFUYBxP5XvRSTi8lcoIn3V68DTp9NnO+xHz7vDJwOTz6fPNnwE1C2Tv9+UNbil6/T8cL0pIj53thwU+/GFdF3/2myrdv4J1evBRPbzkr8497lbcPIT9vfdDR5/NY+uQ28RXT/N39I9YK9f5fbFSy2DLJ11QMt3ueubfG7X29vaFA+lvpB15kz9lxTZLZ4PxYB1PobvB/z2A/p/xXz74AwDYAuIsgBwwQUXYO3atUgkEliyZInNQuDNN9/Eo48+alv+hBNOwIoVK5BIJPDll1/i6KOPtn2vKAquu+461NfXIx6P4/XXX8fYsWNty5SXl+OJJ55AZ2cnOjo68PDDD6OwsJ8eWLdXKkZTx84TlNNLtiXfD5cnN1I2WE43dbeDkClE1h7bF4RllGm2fYGIYGj9DlhjRnWkYiQIuH12AXf/C4DLNgLH3G75vVmG/uo0OVE2yvlzIQJ2FUVz5ovAhFO7Xn9hdffTKKz7V1hDU5IKqsyp0sJn0ir+mv+7fXIKuGjgRWf1owfpvGhJ+j7cQDYT1vOqJaV4BpjiZx5xWwhtibAUBDMCcIfp8+Om7RRUynILu4BE2IyU9dLDn/WYiP23Tl8HZAR6rE1eq2K9Pb0mDN0UZX3OwoaIbnVCRCnbRNmkXRDNlNXtHCkrOpQC0SlPRWWk7IcP2I+HOA6jD6eRWOsotVtM0beKshaRNHvZvtgXeIKyHovrySriZSwc8gxIpBPmtWJeFyLqUNiMiOPTtMI+nVBEEgpLAsFvP6CHfmEFYEUcw3TCHBjQqJMhrpM9TzT3yYyaFJHG1n1yeSniImNfYLn+rBE5mSlcCtUZX7EpImddP6c+ZSmfKi0yxACB8JTVUtJuISdS1hRlswWkTKSLc6RsBndWlLR40NaS0hImI8I20L5MvQb45SPm8nlmADjhCXYdrdZflAy3R8qmonbRWtStWlM0TUUoeiu7k+EN0iCVEHfcPqDleyliWWcWiGN0+GxqDwLl8n4nbAUUlY7pjTW2zeRMrYdC16k7QINl2W3t75dRZCBAHXFvYa6APlhp+gZ49BjqEO9+PA1cLH9ZTl30l9AxiXfIafeGATx4KLD6LVrmj9/Sayomj22snQZ3Cqpk51EcQ28Q6Ky3n/+SYdKuQ9BfUTXnvA2MnZa7PnEfA+wDQdb6OeEUau8BeX8UbfkZL1D0uZhRUjVWtm+NX5MwcejlJDoBFIn4k+s2309vcxD356gZtV46Aph4Gj1PLX+FIs2B3k0n33F/mqIuEG1ztBX49Ang5VkUefXObTR12VvQfXDBEoegk/Z1mQR9Xc5UAkjsUl3S0qA0axC2N8x4pf+n3FqpGE1tfF8Q197p8+g1M9BqAD+fI8+307NFQTUQKIfhLyG/5L1OB35hOe477EN13kqik6Js9zqdBHZrn0pY93z/lrQCEOVzYvxxMop2W+qb9YSp18h9EzMvDIP+b19P135hDd0b9z6T7DScIu9GHkCztNrW5M50KhlOz+eiP9a0nMQywN6WGYZsd8t36n7adV9xuWn2xMF/koMh2Yg2PZ8YPOFkGhAaPx24dx8aZHnyFBIF6y1tSPU4eq0ZT77ugHz+O/hP9Hr/Qfbn6CV/l4OmB/6RXlctJLuOAy+2P5v/6tm+J8HaVvjpHfb6aiU7Ctja/weAjx+xv29aASy5n6Jb370DePmPJPAfd1+uPeHQicBpz5J3beVY4IXf0+Drnqatxvjj6Lre1klmzUw74A/A/r+j+lO7Oz2j5OPGIVSnq8cBh18NrF8y+CKvtwBbRJRlBhmegIwM3ZawirKic+ctpJu50wNgopNGrHoqyooISICmFVvtC4RQ+OkTwFxz5FJ0Mr+eB6x7374uX6FdBBUPBoVZnd3+4nfL7L56VqzHKh87TOqfBrJ6HPk6ASTCJTqk9xpADylWv1fDKsqGzMhm83xlLAUiUjwIllPDXzyEzu/SRyn5TDpBD3zWDq7Vo9FKKkpRYumYKTyqcpuJDrrOVrxsRiZZBJ1MpGyn6QfpMyMWLdeeoiAy4dcWUc5NU5IqdqbzIBLpWOmtz5rb79xhMPT8omygjDq8PbIv8Mip7V11ykQEWdIUlMQxtEY+qi6ZjCnbVzTjKZsnetdlEUFdXpqOJnyZBNb3j03PjVJx+6XIJkRZj18+mGpJaSORgwLySiyQdgGiAyciRoWQFmujaY+Z/XZLT2HrPlePo+0LSxMrGU/ZWK43qxWP6dsspvVne8pavYJF+cyEdhnLC2sdTAv7gi4sBsQ+6Zbfqy46vrZIWW+u2Kyo5gBA1rWUPfgC0PrNBztDlMXjt4/Oi+OspSyirNmOhE1RNlgOjPiRue+9EHSCFVJ8GUj+8KWse9by55THFHryDeoVVFP0pfjc/f/Zu+84qcn8D+CfZPps32WXpTepiqByInYFxd44+aGoiAj2hr1X9Ozd8zx7O5XTs4siYsdytlM6UhYWtrG9TEvy++NJMpm2Owuz/fN+vXixk8kkT2Ymk+Sb7/N9XCLryLjA9+aGyzFY15E9UAQ2jN83o6yA0xs/2yEmm1kD/vV/InjYuL35Y4s7UwQllEC4xnh3IemZxVt+DGdkD5wkAuhGbc38kfrNNE+4hFBGb9El350FPLy7CJCveh94+1zRnVYNiSClwZkufketv9ejjgVGHBnZnqz+kTeDd1TC330psiY4oHdhtN40SQ//HvlqRNDB2JcHHyCW7UwXyzL+B0S33V56coXxuzVwosjY6kiZ/USAYvJN4rErHTjhCRHkLFsuMiUv+LH1y7V2UTeO502VItvK5hKlHZbcJqa7syOzC6M1VgIfXRk+XgIi03DwfiKDDhA3B5rz12fC2d7d2dWbREkS6/tRuV5krhs3vYH43d3nrxDZmu4syAH9+sAIjh54ldjXa4uBhbNFF3tA1Fx0ZUD0VIg6FgUbgSPuFsecrP7ADeVdL9iarP0vC/92/PYaULtNfO9zBuvB07V6MEsS1ww/PSdKZUWTJPEZfHqL+MyCTcC+F4vn+uwuyo4ZNVED9cC/ZoiAmLW3R9F3wN8Gid/u3ATJLKkyYmrz+1R0z7loQw4ETnk1XCd/3aeih8sn1wP/fT48X7wBloxjkLXE3u29xHsHiN/tshXinHLyTeLGwZtzRI3RZAbH7mqcaS3fPOu7h/i/1nItoYSA9y8NP64vEzXly5aHs4/HTm/+N3b4YaKO65gTxLKnvyQCxFf+KcobdAW7TAGOezT8eMTh4v0apd80qNkC3D0E+EMvg/DFveHze+v+58oA1i9tXZZ9D9HNj760Q4yBfjob2R4OsBiZFw6vOJGNl4ET8omL7GRr2Jk1+yTx4/27pd6JUbuwsVKcMDdVhwMCxzwkitE3p75UZJBEjzKeKsagJPFE1zBsS3nDwoNdGPVnjSAfIN67l6cBaz+NzIq0OcX7bx3IK+QTmUiN2y1B2Txg2/9E185Ag+jCtOKdcNdpg686cbDTKF8AiICNK0O/Uy7pwTdXeEAc635g1JQFxPfNyJqN+u7VTbo6nPEo20WXpPQCcQFeXxp7tzzZmrJGTVCj22Q0Y+CmeGRZnMhGXBRoCQb6soUH72pu0DmbS8840DNljewEa1BLsomLQ5sr3NUeAA65QQTgo7u1K9aSD2nhYJwRyLHuy9VFomi8cVG/fmn45ofB7or8PnlyxGdhBDPNwHT0hZAlgGVk21oDjtGZxNHlQWSb+J7J9vhBQUmKHXDAqClrZMpGP2+2x8iU1TNIJTn8PTX3p6gawjanCC40VkaVTtAzVczM7mbK1dg94UxZY/uNoKxZU9YRG0w2agVHX5AGGsXFgDULs3wVcP9IIKMvtEP0es52d2R2h/EbqgT1ALYn/F41VoQvJprLOEpkzPHN19VqC/FuKIb8YruNALSZvRsVvE3PFxlDxm+d8bkbmbPePOCnF2LrmA05QGyrcbHoytSDshmRQdn8UeL4Fl2+wHhsZIU2d2Mp0CA+EyUgAo6X/p543q7gkt9EV1TD2o+Ble+Gb4r23k1kzq7Ua/05vAAkcZPgqvXhjCRXZrjOXFa/8PlG3z2Avc8JZ94C8UvcjDwCOOTayGmeHFFmKFWib2hLlhp3RjAh3sW7EVjy14rtNF5jfN9kWa/XavnNPO3NyC7v1xTtfPtTweEW9R2jM5mMzyQUEBm/O2PQfvqgcZX6DWX9vb12i8hgzR0aWdc52vf6+WewMTxgT1OlOJ/y5omL6ZbqDbsyEpfA6k4kSfxu5gwWtYwv/UPcLB16sHh+t7+K5+LdnLM5zNdL0eWUDr1evH81W4DVHwHFP4vf3aJlYr8cOAkYFdnrE0c/IDLmNFX0bmir64POYuQRIov+t9dElqe/Xny/f31F/Bam5YubT333EN3+94gTlAVENnd9iegZ468Dxp8qjiv9/xIugTDhrPD8uUMjf8sat4v1NFWKhAkjqNsRsgcCNzWzbxuM89jMfmIbL/4l/oBnhvLVImGo756xAVtjzA3jfO/M98T/xu+2UW6lpznomvDvfLUl4/11vZ51yA88trfIdLX+Hl9bDEz7Z+ISXVZjTxafn/Fexxu8tLPKHxE7UB0gxmzYY6bYr5oqRdnCYBOw9A7gjjjZ1cFGUSvZ6AlEJgZlKZYxoFBnI9vDF4PGCZFTD5RaT2ZkW/jE1JmefKas0ZX6lmrxw2zUIgLERXFavggEefOAuweJ0SztblF03yhGn0j2QOC4x5JrR6oZ3dHauz5wuh6UDTWFL8pc6aILZ83myMHFjKCsNzcyUzZnsOjyZXQHyRksRsjMHylOUowu00BkhkF0jUYra1BWtou/67aJdRiZr+NPBc75KiqLLxQ+ibG7xPoUf/ygihnkshykC8fq9VGjbiBYMyybY4zia3eHB9mxUpWWs1yig262qP0G0LvphyIzQeOxOfQMST2r1hih3Rp8M2qrGm02lnfQleH6w9b5jWxLQHx+Zh1Yozu+pf1Veg1C6w2J6O5udrf4zZD0IPlfnxUXP4r+WTZVtxy8c3iBle+F67wa7bS+Pw0VUUFZezgoHy9jW5LjZMrqQVkjMB2dcWpwei0DfYXEstxZQMEYS5axpRSF8f9xjwLHPqKXIbAE91yZ+m9mAM1myhrlJsxMWYfYH40u9qqe5azGCcqqccoXBBvF/h7yibqMW38Jvyd9xomRyYFwRrXVmR+IZRo3dozn68vD+7bDGx5sKlne3NgRaNtavBuKwQbRflkWA6kYv4nRJTzSCkRQNrpLucGbB/z6sj6YRDMZWEYtXVd6eHCx3mPFPubwxH4XjUC6wyhf0FxQtlFkkmcPFMEmo2ZjV5UzWNSFvN4SvLYG8r15InvGCHB5c/XeR41iHzIzLjP17+gwIG+4uDF4xToxb/6IyOyZrDasz9mczD4iaASI2ri5Q8LfV+P/eAOn7HKYKAGlhsT2WGtFGrL6i0yZRCVoduSmSnu76GdR93tnHXItsP988R1oqhIBPEAcU4JNou71hq9EXcz1X0S+dvVHonbskAPFDZXb84Af/qnf3Oojuo1bbyIk4s0Tmcw9hSSJLrfZA4CTXxDZlIC4QTJ+ZvxMWcOwyag8Nk698fQCkckYagK+eUh8BpMuFL97e5wWW6P3L3PCvRpaymTuDvY6U9TC1RTg/cvEscuauXjQ1eIcbchB4v1IdP5pBLIaysM3frIHit+ay1eJgZKOeRCYu1TM50wX5zxGzyrjN2fz9+J35vDb22BjWyGZYN6hN4ibNA1l4jhvHBMu/Clcp9PKOCYbdfGv3iheM3a6eG7rL+FMWqPc0UHXAPNX7dSmdGmHXCsGQzzldfHd+vklcV26ZpF4vmKteP/LV4lzUENrBtjMHyEyn7sj43sZaIjstXhLljgXv26beDx2emRpNDJ1YKEm6rRkGxDcgUEF2lq8oGy8TFmbUw9a6RmvVRuTXIE1UJERmbURbBBdPI2TiNot4kcn2Rp5HZkdJMsiG6C9pfcGZi8S9XaMoKyZoaoHYY2Dmd0lSgd4csIX/CG/CCj7a8OvyxksAqhZA4Di/1qCW1EBh5rNIkgVT8gfDoTKDj0oWyJOzny1QJZR69EdWe/SmtlrXDAaGW3RrEE5Q+9dRWA/untast0FzYG+nPEzZTW15VGJo0/+4pUvsLnC3XKby5Q1utgbgdX6EjFIyGGWE1wj69YoGWBdnqbpQXVLsNPIEgXER2oEoYxp1uCcUb8od1g4EBgdrLY7RVDIKPlgrF/VP8uazeJkvmJt1MbppQsAEQRd96nIgDO+m8ZAX8b7rSlxgrJ6jdZ4WYTxPivZrgdvmvTAZ3PlC+r0ILGe8dtrOHD+Mn3APsv7ZX6mztg6V9Dff6OGshJsoVyFOzZTNm+Y+Pfj0+EbJMZzRoDVzJTV3/vtf4rXGEHZYJPIeG7cHr9OYHRNWUDPvA4A0AeuUUMiY7ZuW3igFEkS2SSdXbzfj3GnAIP1GnOZfcNZMtEZk+kF4v00shWju+ZFDK6YoETCFWvF96mpKvy9BYB9zhUB6s3fx77GKGegqXowv5nfCU0R38t4dd+7Moflc7OWuTF6RPz1OVFDbejBwIvHx7wcNjtw/TYxerOxbybKmukzDrhqQ/zn2tIuU8S/314TtQ2tjO9tv73ERazVrieI794Lx4j5nGmia3b0sg+5Lk694i7E+P1LBW8u8OrJoiTFcY+Gy2f5qsX7N24G8O0j4kbM0IPCr1v5vvg/awCw9E7x98d6L4MRUyMHMmyOKyM287qnkKTIc6HdpjU/cJs7C4F++8ROt3vCWYiA+O3rvWvL67/o58RjQnQ3nlxxnlBbLHo3OdOB60uABYUia9V43y/5LfEyjPrIH18rfm+n3Bp/PiND1JUujm1P7g/sc7445+tqjGuo6s3ippfxfY0ebNkw7/NwZjcgrnMu+0Mcu18+CXjqYDF90P7h8wbjBkFPJtvE+/HpzeKxcUPR2wt4cj9gzzOAn18Ux70LfgT+Pqn16zCuZ7ubnEGifm5TNfDv2ZHPeXuFr4Uy+7CebALMlKVYki1xllZHMsoXDDkonBHk8IofAGuGkLV7t6sVmbLNCTaJC+DaLeLk1ZkRHiW9K2jLkXATkWVg0KRwjUzAEpR1WMpFIKoGph54bKoOBxWM1xkHyOyBIpPBHDApTsChuUCBcUEp28Sya7fqAzrVhttiDOBmzKvqXbBlRzgLyhpEtNCiB/oCxEnV4bdFznjBD4nbGLvUcNt3dATz6GzpiExZo6u7vo3RpQVilqV/pkbW6IxXxWjk1u76Rm1dI4gWHYhUg3r3d6PGqz984NYQDsYZ7bTWGf7tVeCU1/TgsBIO/hk0Tc+UbQgPjibb9Axn/bMMNMTvGhxvOyPKF+jrs74/1uVYP/94mbJqgqCsMaBLc7/BRvkCo36sdTnRGcVmTdkEWfJG+QZrCYREjJIKshxZysPcppCYpkVlbBuBeSMg/uieov3BJnGBFvKJz0rTIkeiN9jdsSMx2xx6bwh9n9BU8b74qrveCW+8719mX2DAX8KP/XXA1DvFBYFVWr74fTVuMuWPBk76Z/h560jeibKg0wvE73B9mb6/6EEym1OUoImXBWvcrFLV5G5MOtyxJRC6iwt/CtehA8L7RVov4Hi9d0xz3cIz+wMZfcXgX4nqSUpS6waSSrXogCwQ/twzCkW35GhGEMHu1n+Lo46Th90qfpdjbhb1UMbvVs1msV8bwZaTnxOBWuN9iv6dNkr8ZPQB/tCzPY2bk/HqTFLL8oYBu57Y+tcZx73JN1uWlSBoFr2+7l7L15BeIAYHOuo+cUPRlS5+K/a5IPl6ujYHcNYn4m9fTfM1zQ+4XATLjf3nuydET4Yj7xV1brsaV0Y40H/GO4nnszvjv592l+jlaTjt3+3fi7Kzs36fXpkmspRP/494vO/F4gap0aPlhjiD0fVU2YNFXfhfXhLlSXY9KfxcdGIFxcVMWYplli9IkFnTUWwOEWA57tFw1w2nPuhNdKbsPUNEVyxnevN1d5pjd4XrxgYawhdFmioOitt+675F+VMpf1T4gtw42ClBoCpqgBpArz8bFCN2r/88XHMmvVAMWGEE7LIHhoPtEUFRTQ8AxskkBSwDi1mK63tyxIWQJzdcU9aYp7EyfDKn6DUzbygLn0AnzJS1xWbKAiIDI+K9GRm/nfGMOEJsd6KaskZ2a3Niyhc4Evxtjx2YK5oxv5EZaYz+GtEm/X2wO2MHezIyHB2eyAO2wytG363ZEv4cjXUZ2731F5GFdtR9Yt01m8NZuwajrIISAtxp4rWSTc/QDYVHDTe+P2sXi+/jF3/Ts/r0fdu46LVbuuYby7a+P9a/jbINRvZrNKOrv5UsixNuYzCuRAN9OdPEb5oRJJWi3lPAUr7ACLTHO+mWwmUImiuXYF12dKZsxDbp01R9HuifRbyBvhoqwr+pQZ8Iproywp9vn3HheY0seuv7GD3iu6qI55vQ9Qao6L1buEt7ImpQ/D5F3yQxBkowMjWdXmD36eHnrV3Am7twNepq5wwK/1Yb379431/juYJRsaMUW63/XCwz0YjH3UGiTCVrlvKlfwDrFsefb/qLkRlNXYWjhWC8eYy1J/4tozDj/LKxMvK7YJRTKVom/g82ivOnLT+K/dbIqG2PMQMoOaOOEb0dfn+jded4PYEnWxzvtvwkMvAH6APRHXFn65YzcCIw/HBg7SfNB7SNY6v13HD958BR93bNfeaC78Pnc0Yd5Naw/rbkDY9/fO/pjN4P+1wAfPe4OK/tow/kmbdL5HuYTOmJnmL0seL6ZNihInu9cKy4qfj2+eGyWNSsHnJrjlrFvEDvZBcJxuA5dlf4IByvlp7xd9VGPYCRRKbszy+G68YYMvqG68o2bhcF6qe/JE6Ic4e0ro5MT3bE3cAxD4u/jc8r2Ai8fpoIrgHhQOjoY4AT/yEK9m/8KtydM3cIcLblotaTIz7XUJN+8Wf5rqbrhcVjApSWx8b3x+4WWSmVG8ODwhknPHZX1Mji+kBb1hPARNnS8WrK7qw+u4uMJWd6ggOcpct9sqxZcNa2GlnMzXZn17fbCCLGYwSnjcGaIrJD9YxVhyfyhNmZBhz3CNB7jJ75ahmQzMiUNQb8yigU6/j3HPE4IlNWr3+qKZZMWT2QaAQQQ5aawG/OAX77lzhhtzJGvTZKZZSvFtms0TV3rdtmlq9wiEDZ9SWRy4yXaQpEBs4SBdiNk2ibIzYoa4guX5BoAC8jU7alOuLWUiDWmrJWxjTjfTfEqykbqA9nyjZVieChNVP2yLsjt9dXGxWUdUYFZUOR70tX4vSGB39KZPpLoitytIxC4Lxl8b9LgOgidk2RODlubjAuu1vvceIR+8lR9wG7/594zuaMLT0gO8QFy64nNl9PNtQk6q/1GRcZaO/ucoeKmqkGWY7/+QGJM5o6s2MeEr2FmmPurztwXOqJPDnA8U8AdVvjP2/cYAnUixHUnztSdMfe/D1wzebIjPvDbhOlhKhjeHPFb+9+l3SN2sgdoddwUYIs6YFu4zj5+eTnNcbWAESZo64YkAVSc37Tb4L4/6L/7vyyuiO7Czj3a3GjYO9zwj1hri3uesfq9lQwSvTKPf0/4nrVeK8m3wyc9I+ObVsXwaAsxZJauEDvKEZNWWuGWfSo00D4eU2LDcpqGlAeNUI7ANRui52W2Sdct7KhXPwwjzkuPFp7XjM1pyhMlsNBA6clKAuEg3FG18bsgeI9zh8lAmA5g8WFijXIM39VOJPQyF41Lvpku7iTeeFP8dsSfUB1Z4v5a4rEiXRE+QK3HpTVL3biDc4ELfmasqnSb4IY7Cz6QjeZk4XoDOJEXfen3CLuwjebKWup85poO2WbXr7AGZt5a3OI70G8TFlj+dHlC9SQCJIGGoABE0X7IwYKswRlVUtWp1GXVJYt5Qsc4dIKgOgGF+9Efb9L9OWpIuD4+N7Axq/jZMpGBWWN2rBA8tkIxo0eo35wPMb7Y+wD8YKyZvkCIwidoEa4tVxCc7/55y8L/91spqzNkilrmd8oF2EEmv31ek3ZPGD72nDPB8Uv9nfrtttdYr+0XrzZHLGDXsUpI9JtjDkucff13glqZxvcWS2X6ZAkfUBG/bfMmRb+DPr/RQR2rQ6YL0YSt3uaX+5xjzbftu7q4l+6dzBmwuyWu1ubx0UGZJM26ihxQyQe4/fNGFDT0FQpegc408TI3rPeFwPRWW9sUfs58amuV0KnIxg9WnamfElrArpTF4j9goC5SyIHkqRYhWPF/0fdE85IZiLWjsnozd/EJLF8AcWS5U4alDW64louvo16jdbAmPVvV2Y4qw4QQbbH/yJG57TK6i8KwFtl9BF3VAFR+8jMfJFEMOSUf+3wpvRYRgAp2CSyiYzRaG1R2aZpvUSmsicb2GNm5HNGgXAj2GfNxDHqwMbtUhqvvpJTDEgGiDvpvurImrKNleETv7hBWbSQKdsGP7GyLAaSiX7PkET5grT8yBIK0XfdjVHEjUBkXZybFWY7rFmnCYK3kk0f5CdOTVmjrrAnJzLA5rQEZUP6gH1GO301ojTJcY+J2k6AXidWX65iybg1MjZVRQSbjIG+JDk8CFcoEFnr0BhQycgGhaRf8Kbr5RH030V/rV5f2FqywFpGIEH5Cuu2RwcVgdis7HiMAK8xmFa84K1ZvsAGXL46/B2PduwjYhlrFydfR9yoERtTU1YJly+wbrckic/ekwuU6gMOBvSgbFovoGKN6O60fZ2oGR4d5MvsH5spKztEhq3xfhl1dim+QGPz5QsAvReK/h5by6MYgwtaGTVGN37dfFC2uw3uRWEtDSppBKUbt0dmqVFinhxg77nxn3NliBuyxnFj0oWiNqeROLDLFNGbKyPBbz21j3H/19Et6Dqir8NaS5KSq9lrzLt9HdB7LDD5xp1bb3cwJs7gk0TUoRiUpVideaAvIH6mrDUwZlwkjj81tltudVTGj0EJAEMOjJyW0Qf4fSHw8wsQ3db1AIxs02v8Ze/olvRcvUaIi4lAg8iG7TNeTI++6Jck4PKVzS/rz8/Ed2HYoTADWHKCYFc8Z38m/jeCSwWjgV9fDrdFkkRwwghmSFL8oGy87MpENWVTZdtvwGG3R05LpqZsRiHw12fF36OOiQ3WGfXrDM2133gvlEASmbKu8GODM02MJJvRJzKT07jRoqnAPw8Rg5zYXeIGi5H1HmwKB+dle2QmrbV9kg1mnWFftV5ywAiYR2XKyo5wWQjZJoKGxmcbqBf7u7HNTVUiKBFRzzW6fIEvcVezRN9TZxKZssZNJymJTFlJEp95IjmD9PmTGOjLINnCg95ZmdmwUQN9GQOw2V1AXamooWoM9OXNE2VmsvqL1/pqwiMrA6g85nlka1WQ6ksiA3w2p7hhYAx40X+CKDVTsTq5behpAvUtZ2sH9UzZk19Ivg6i3dN8thLrrXVfLQVljayiI+9BpyuF1RUNO0RkbN2aLR5PXSD+N37fjbJNRD3JRQl6xcXzl7miDjsHwCOiTojlCyhWS/UFO0q8kcSNC3VrdqzR1eDQ62OXUbNZ/K9GBdeMAb2sMvuKGpMbvhSDUhn2OB0YeVSrm08QF/sTzxUBL00LB29isj6TMONflm7vejDS5ogM8K1bAnz3ZPzX998r/PfN1SJLxVcbGfTPHxkO8gd98UsVxMsUa4uaslZ7zhJdSCO0snbfjFfCNXsTiZcFbDCDssHEn59Zt9UIdFvej/RCoHqTuHiPlylbbxnV1OYUn49R9iLYaOnGbwfS8oAZr8YpX2DUDrbUlJXt4rO0OSIzZR0eUabEWKb18z7qPhGwtgagjWUBwF6zo4KyLQTlew0PD3BhZXZRbyZT1gjWmpmyzdSUTZaR4ZrUvHpPirg1Ze1R5Qu0cFDW4RFdbTP6iCBhoCE8GFJWf5FZV1scsT8F+k/Ss9+ja8raRYaY8foptwDHPNi6be5J8oYBpSuanyfkF/vUrifEH7QvHoe7+Vq1NifL/HRXLQVlAVFLO7Nv+AYa7Rzjt9+ocUhEyRt9DAOyRNRpMShLsZIZibsj2CzZXwYzqGcJfuSPAk56Ovb1S+8SmXlZA0TWnFWwMZyhZ8joEw76nfFuePq4/wPG/nWHNoEgMuEayhERQGwu+JeIObq15fsQPQDQd08Ai64OP040MJIkiQCDvzYy6N//L5Y6uAnqMsbLnjWCem2VKXvcI7EZuslkyrZWszW79HVZg67RJJs+mJT++Vrf+/4TRLajzRkZlDVutFiDsjmDgYOvCZcXCFkCpsZ77fCI5agq8OMz+gBklgHdQn7RHtku6mfKdksJDIjPvalK3zTVMogcRJfS7IHAqvfDGaXWgb6OfSgy4NxSUH63k0RQ3HDJ/8T/ab3EoE6a1nLQw+5KPJ/NAVzY3CAO0fWIbbHTEjFqxMaULwhZMmWt74X+HbC7RTmQjMJwTVmjrlzWAJEJXbctNsgn2+PUlHWKrFrrNIe3hW3uwQ65XtSlbU6iQQubY3fHHjetRh0tRoum7ieZoCxH9m4b//dyR7eAiIiIUohBWYplDOTS2TRXuy5iPi+w+8nhx0YNxC/+JsoX9NszMuADhAccil4OAMxeBOQO2fF2UySHO1wr1OBMa/7iPh6jZp1s+b5aA5M2O5A7TNSkNGqNhnzhwWyi2d1iOdasz/GnAn31O+vHPQrsMjnyNSe/EDkyvUG26QPndPGL0niZwYaMPsDAfUUQLmGmrByuKWs8NhjZPtZA+sHXhgN1BaMsy7GJoGjAkilr/B4YAVCHV3x+gXrgg/mRGZtmpqwtMovVGjCWHWIQFdkhgrwhf+T2Rw96ZmSqms9Hly/wx2aTJmKUEQBE8CxRrVgruzv+fKe/LfaNXq3IUGxNRrdRviVmoC+jhq9qea8k8b6oevkCf524KWNkyhojuHvzxOdeuy229qkZgI2qKeuvjdyXJal129yTjJgqAqQtae1NnfTewMgjEz8vSa37blEXwpIEHeL6Esv4BkRERNQdMChLsWQ5tnt/Z+DKjD+9pVEkvbnhDLgf/iEGS6gvjZwn2JS4G6Y7wXppx0Vf/Gf2Ba4rbt0y0vLDRf6NgJh1VHjZIYI5uUNElisQv0yFwQjAWbt+99k9HJDf8wwRGLTa9YT4o6Kb9Y+TDMqlgq82fn3bndFcN/jeuwJnfST+TlhT1h6ZgWcNXBqZzrI9HFQ/+JrwvJNvEvVkrW0JxPkcjYG7jEzZkE9Mt2ZsOjwiOCzJ4aCs3aUPbmTJlPXViM8z1KQHbC2fX3TwM7rLf0xQtmknMqWbKV9gsLtglgewGnZIEoGwONuSLMnWQvmCqIHfjPY5PGI/9OaK4Ky/TpSuuGazeG+daSKDPvrmm/G5RJQvcIQD8ZQamtL63ytXuugSSj1PMpmylHpd/UYvERERxeBZFcXqrAN9JQo4tTSKpLeXGMDH6Bad0SdcO9JgrVFplTss+dE9KXnRtV+BljMDo6XliSL/aiicmWcNyhoZdukFors0EDnCeDTjYicVgdS2KlvQnLQ8IHdoapfpygAy++3462PKF1iDdRJw2fLY8gURNJjd6mV7uHxBdE3ZoE9kWv/4T+Cjq8R0a6aszRlZU1bxi8/bX6+3TQtP9+bFZnIb7Y14LDcTlNUDvzv6XdK0lhPR7K7ENWVbXkHkw9Ysw8hMj8mUDYnt1RT9OX0dZrayW2Q6e3IsmbJp4ZtedrcYqT06KCvbRQDXHpUVa7yGUiOzvzhWEiWDQVkiIiKilOiAyAF1ep11oC9vL+CIu3fgdXkiCJvRW4z0ndkHeGNWZF3YQJzyBQBw8c873Fxqht0tAqapoIbCwZmDrxWBV0CUL/DViHqk/joRyFECiesmGsvYkfq20ToiKHv2ktSv15UOzG9hgKDmmCUC9HZFX8hn9ReBPGsd4ERsjvBAX4HGqJqylv13m16f1Zopa3OGH0t6wNSdpQf7XOHlAIAnVwQNWwqqSlLkzauIOqo7OdCbUQagObadCMoGmyJvQrU2U7a5mrJqKLJN0ZmynlygrhSAFlmyxO4SA4FF3xyTHeLziBeAZdZY6lz8c+sHiKOei0FZIiIiopTgWRXF6rQDfdmBfc5t/evSeolyBUbG36D9xCBDmhbuRp8oU5baRtUGYMuPKVqYpY5sVv9wXUmbUwRzvLliYDdjwKFEgTYj6BNd03KHmtQBdRRtjtZnG6fCUfcBfcfHf062hTNUgfjBP2v5gmg2RziLVnaEa8oG6sMBObOmrP7YGMTPmilrlCYxB/ryiXIovho9ECWFvxfGcgpGi6z6aLPeD/9tbXfEAITNbG8ynGnhrOBEbPYdD8pu+AJY81H4catqysoJasrqZQs0VV+e/n6YWa2ucKbs6g+Ams2RNYaNGrnRbbE5RUZzvJslzJRNHburY34/qGtiUJaIiIgoJZgpS7E660BfO8qbB5StEN3YgfCF52e3iyDBkX8TAZz2rP/Z0x3/BLD5u9QsSw3FrwcsO/Tn0kXwzcjwSzT4khHgSUVd1o7IlO0oe89N/Jwk64G2ZoKU1oG+otndQKNeD9rmCNeUDfnDwTsj2G4E7YxMaU0NB/2MjExzoK+A+F40VYVrs1rbaHcBp7wWG6QaezKQbxmATFUQDj5G1ZQFkh/oK5ozPbyt8Vy1QfyfzIBg8Yw+Flj5Xvhxa24iGPNaX2N8NrI9aqAvhN8Xu0fc/DJq+FYXRS43UYa6zS6WHe953kgj6hgM4BMRERGlRA+KHFDSkuk625U4vGIQJLtbDCpjqN0KVK7vuHb1ZAWjxL9UOPEf8QPqRrd4VzrQVC0CRpKEmHqaBlkGjnkwNRebPSko2xzjfZAsAdRo1mzYaHa3GDDLmM8YsM06vzm4lL4uI8BrzZQ1BxWzhWvHOtNE+QLZLjKtjaDfLpNF8FCOkwk27enIshvWQa3iBmV38Hsw4giRUZqIMbjcjmbKHvtIZJZpq8oXyKJ0ghH8BkR2caA+3MsiopSDMdiaWwRXbU7g6PuBD6+KXG6iWs9G+YJ4XesdzJQl6hDMlCUiIiJKCUYOKJYkd87yBTvK7g4HDIxBZQAR5GmsBEpXhEdsp64no3f86Uawz50F1GxJPECR1YSzUtMmBmUFOSoYmyhTNlFmvsOjD7oFPThnBGUtmbXGTSQjK9VYlhEcvG4bULFaTDPKF6iKyK4ONuq1YdXwYGR/Obv5wLzNkrFpzQK2Zoca2aDxArvJyB4g/rVkR4Oy3lwRYDa0pp12F5A7JPK31O4WJQbMmrJxMmVtLj3Y7RCZwNHHmERBaJsDgBa/VEGiQC4RtS1myhIRERGlBCMHFMsIXnUXdldsTcLqzSKgMfEc4O+TOq5t1HZsThGULRgDrP5IBIq8eSJI29Z2dICn7kaKCsYmCsomGvTNbsnINLrIO9OjgrJGdmZUYFEJiICg02tZvyUo60gLDxymWcqXtBRssGZsjj4WyB+pv86yfmca2k0qMtZaswx3NnDOl5G/pw6PnnVsi8weti7b+PxsjvhlBzL7xl+f8bnYmSlL1GkwU5aIiIgoJdr0rKqyshIzZ85EZmYmsrOzMWfOHNTXNz94ic/nwwUXXIC8vDykp6dj2rRpKC0tjZinqKgIRx99NLxeLwoKCnDllVciFAoHEd966y0cdthhyM/PR2ZmJiZNmoSPP/64TbaxW5LtIpjVXU667W7AXxvZxT17ILD2k/CgUNT9yA4xIFH2QKBygwgUnfUx8JdmaqCmSnfZd3ZWdDA2XlDWnRkeiC2a3Zopa08clI2+iSTpdWPNoLAjPN0IHDo8lqCsmnwtYWvwN6MQGHKgvmxrUDYFg8UlKxXfNS1BSY9oww4V2xxd39X4jTUC3hFBWf1vm0OUkXGmxa8BnT0AuKkydrrx2dni1JR1ZcZOI6K2x2McERERUUq06VnVzJkzsXz5cixevBjvv/8+vvzyS8ybN6/Z11x22WV47733sHDhQnzxxRfYunUrTjrpJPN5RVFw9NFHIxAI4Ntvv8ULL7yA559/HjfddJM5z5dffonDDjsMH374IX766ScccsghOPbYY/HLL7+02bZ2K+aASN3kpNvuCpcvMJz4JHDZciCznxhh3TqiOnUPRk1Zu0uUp5Btor6srR06CPTbEzj367ZfT2cXU74gTuDVlSnKiMQNyroia8pCEwG9mPIFUUFZd2b4MzdfC5HlagZlLYHBVAz0Z21/V8uUTZSpHO30/0SWLTDYnEDQF67PG698gewQA6s50xKXKkhUcxiIX76gPbLeiShWsjdyiIiIiKhZbRadWLlyJRYtWoQff/wREyZMAAA8+uijOOqoo3Dfffehb9/Yroo1NTV45pln8Oqrr+LQQw8FADz33HMYPXo0vvvuO+yzzz745JNPsGLFCnz66afo3bs3xo8fj9tvvx1XX301brnlFjidTjz00EMRy73zzjvxzjvv4L333sMee+zRVpvcfRijmbdmRO7OzO4WXWutQRdPtvjnrweyBwFDDuio1lFbke2AogfrVKV967zanEDh2PZbX2cVXUs23mfgyQEayuI/Z3eLYB8Qzph0pomB28x1WDJlZ70PvHCMuNkS8lsCgkY75PD8Dks9Uk0Nl7fYUdbXphcCxz+x48tqjVQEZXvvChx1346/3ubQg+COcC1fSdIHWzPKFzhFgN3hBQp2BU55LfllA7HlC/7vZda1JOoo3anEFREREVEHarMoxbJly5CdnW0GZAFgypQpkGUZ33//PU488cSY1/z0008IBoOYMmWKOW3UqFEYOHAgli1bhn322QfLli3D2LFj0bt3eHCfqVOn4rzzzsPy5cvjBl1VVUVdXR1yc3MTttfv98Pv95uPa2trzdeqqtq6je9iVFWFpmmW7ZQg6ZmyWnfYdpsDkr8OkOyx2+PwAif8XQwURN2LZIOkhqCpKiRVlONo6+9z7L7U00mQAajG/5Icu69l9IVUtQnI6BP7+ch2SGpQTJdskAFoegkCc159uqqqwKD9xDw5g6GFfCIgaHmtqv8taQq0pmpzmqQqgCRBu2JdUr8F0vCpsW11ZUHabZreVhkYd0qb/67IAFRN2/n15A0X/3Z0ObIdUqARmiRDUhVokCFJMqAFoBntlG3mdwE2JzB8arPrM/cl6K+THZHzjzyav9tESWiT45ISDP+mEvUgPM8jSg3uS9Tdtea73WZB2ZKSEhQUFESuzG5Hbm4uSkpKEr7G6XQiOzs7Ynrv3r3N15SUlEQEZI3njefiue+++1BfX4/p06cnbO9dd92FW2+9NWZ6eXk5fD5fwtd1B6qqoqamBpqmQZZlSIF6ZDXWQnOkoaasrKObt/OUAPKbatHo86Mh7vZ4ge6wnRTBW1uFdNmOsrIy9AoG4G/yo66NP+fofanHC/lQCKCiqhYFACq2V0Ftis3Az22qR8gVQm3U5+Np9CNTCaJMn14IIBQKArBjuz7NXVePbMCcx33oPXCW/IzA9jLIDU1oLCuD3FiNAn0eb0MT0pUQyvP3hTRzKdSyMuQG/Aj5gzHrT2jyI/F/M/a/s11/SwoB1NbVw9fBv1/eRj88vjpUV1ajlxpCXUMjXIEgpIAPddU16AWgpt6HHIQ/p5YY+xJ8EvoAKKuoYmYs0Q5oi+OSo6IMeUh+fybqLnieR5Qa3Jeou6urq0t63lYHZa+55hrcfffdzc6zcuXK1i62zbz66qu49dZb8c4778QEia2uvfZazJ8/33xcW1uLAQMGmIOFdWeqqkKSJOTn54sfxUAaJLsMeNPgauY96zI0DZLShLTMHKR1h+2h5GTmQMoegIKCAkiyBG96Bjxt/PnH7Es9nRIEAPTq3Uf8X1AIeGN7LEg2GQ5vGtzRn8/WXEjQIn677Q43IKnhaWVieebjgrmQFl0Nd5oLsGUhvaAAaLSH58nKhqQpyO8/NLx+ux0Ojzd2/V1AZlY2Mju63Vk5kLQg8gp6A2oIGVnZkCq9gOZDbq4o65CVJ9rY3HHYytyXskV93oKom7FElJw2OS41ivPiZPdnou6C53lEqcF9ibo7tzvOeBgJtDooe/nll+PMM89sdp6hQ4eisLAw5g56KBRCZWUlCgsL476usLAQgUAA1dXVEdmypaWl5msKCwvxww8/RLyutLTUfM7qtddew9lnn42FCxdGlESIx+VyweWKHd1ZluUe8UMhSVJ4W236qPWSDVJ32XYlBMnm7D6Dl1HL9j4b2PN08R3WRE3Z9vg+R+xLPZ6oByrbxW+rbLPH3wfVYPzPxybqiFrfS8npBQJ14Xn1mqMR77fsgKQERP1YWTbrkZq/b0owcl1qALC1z/cj1WTZ1vG/a3pNWcnmBKBBku1mVqukD6wn6zV8W7NfSJIU/u509DYSdWEpPy55c4DsQdwvqUfieR5RanBfou6sNd/rVgdl8/PzkZ+f3+J8kyZNQnV1NX766SfstddeAIDPPvsMqqpi4sSJcV+z1157weFwYMmSJZg2bRoAYPXq1SgqKsKkSZPM5S5YsABlZWXmHfrFixcjMzMTY8aMMZf1r3/9C2eddRZee+01HH300a3dzJ5NtokMt+70A6kGd350depabI7wZ66q8Ud2p7ZlBk6dwF5nioG74tESfD7+2tgBwDzZQGOlZR1xDmM2uxjoy/j8nelR83ejkcM7Q5d+mwMINlkGdLPBfI+NASMTffYtac8B+ogoOYW7AZf+r6NbQURERNTltVnUbfTo0TjiiCMwd+5c/PDDD/jmm29w4YUXYsaMGejbty8AoLi4GKNGjTIzX7OysjBnzhzMnz8fS5cuxU8//YTZs2dj0qRJ2GeffQAAhx9+OMaMGYPTTz8dv/32Gz7++GPccMMNuOCCC8xM11dffRVnnHEG7r//fkycOBElJSUoKSkR9emoZZINUALhi+nuQAny4r4nU/yALTYTntqJzQkc+zDgSBCYszlFEDVa3z2Bw26PnHbwtcCs98KP4+3XskN85kbA0nqDKd7vmmQTgWHaMbLIlDUD68Z7LEli0DNgx2+KdYagMxERERERURto01TIV155BaNGjcLkyZNx1FFHYf/998dTTz1lPh8MBrF69Wo0Njaa0x588EEcc8wxmDZtGg488EAUFhbirbfeMp+32Wx4//33YbPZMGnSJJx22mk444wzcNttt5nzPPXUUwiFQrjgggvQp08f898ll1zSlpvbfcgyoIa6V2ahpjBTtidTle6V+d3VtPRb4vCIoF60/nsBk86PnObKBNLyLMuOF5S1A6GoG0u31CSev/9fgF4jmm9jZ6V1gqxfmxGUtWbKQrTNCMoGG+O/NhlHP7Bz7SMiIiIiIuqE2jR1MDc3F6+++mrC5wcPHgwt6oLS7Xbj8ccfx+OPP57wdYMGDcKHH36Y8PnPP/+81W2lKKoavpjuLuyejm4BdRRJ6lY91ruclrId7W7R/b0l+5wPuLMip8ULttvsgL8+/o2YeAHio+5ped2dVWfI8DXe5+igLBD+fPruCcz7fMeW/5c5O9w0IiIiIiKizor9uSk+Tele5QsAwM7u6z2X1DmCVxRfokzZaEfcFTst3s0j2QEEGuKXrGAZk9ST9aCsEZyNV77A4Qb67tH+bSMiIiIiIuqkulkqJKWMqnSv8gWACPxQzyRJYKpsB5n9Ucvz7EzNZzleNqxdZN7GzZTtZkHZzlK+AAgHYK3Hju7W44KIiIiIiChFeLVE8WndsXwBM2V7LIdXZE5S+xu0b8vzKH5RwmBH5AyKnSbbRQ3TePt8twvKdoIMcOM9NcpUSLZwsLi7HUeIiIiIiIhSpJtdnVLKaEr3u5je0aAPdX2D9gVcGR3dCkrkxKd2PFia1R+4qTJymmwTQdlka8p2VftfBuQO7ehWxL7P5nssdb8yOERERERERCnCoCzF193KFxx6A5DRp6NbQR3l+MdbHmyKOk5G7517ffRvlZEpG7embDf6XZtyS0e3QIguISGxfAEREREREVFLGJSl+DSte2U4HXhlR7eAOhIDsj2LbAcCjYDNGf85Si1b1HtqBr41BmWJiIiIiIgS4NUSxdcdyxcQUc9gDPRljxOU7QwDY3U30Zmysk386461yYmIiIiIiFKEV0sUX3crX0BEPYdsB4IN8TNlFX/7t6e7i36fJZsIxqpq5xiIjIiIiIiIqBNiUJbi05TuVb6AiHoO2aaXL4hTUzYUaP/2dHfRA33ZXeL4oSmAGuqYNhEREREREXVyDMpSfMyUJaKuyihfEB0sBJgp2xay+kc+tjlg1pNN7w2cvaRDmkVERERERNSZMShL8akKB8Qhoq7JKF9gj5MpO2h/YMa/2r9N3ZndBdy4PfxYdogMWUkGZBnoP6Hj2kZERERERNRJMShL8alBBmWJqGuS7aKWabyasun5wKij2r9N3Z3NcrywOdnbgoiIiIiIqAUMylJ8aih+118ios7OCAbGC8pS27M5WJeciIiIiIioBQzKUnxqiFlORNQ1OdPE//HKF1Dbc6axBA4REREREVELeMVE8WkqL6iJqGvy5Ir/+RvW/m6sEJmy+17U0S0hIiIiIiLq1HjFSokxoEFEXZE3T/wvSR3bjp7IKHszaN+ObQcREREREVEnx/IFlJjMmrJE1AWl5QGX/K+jW0FERERERESUEIOylBhryhJRV5UzqKNbQERERERERJQQg7KUGMsXEBERERERERERpRyDspSYxK8HERERERERERFRqrVp1K2yshIzZ85EZmYmsrOzMWfOHNTX1zf7Gp/PhwsuuAB5eXlIT0/HtGnTUFpaGjFPUVERjj76aHi9XhQUFODKK69EKBSKu7xvvvkGdrsd48ePT9VmEREREREREREREe2wNg3Kzpw5E8uXL8fixYvx/vvv48svv8S8efOafc1ll12G9957DwsXLsQXX3yBrVu34qSTTjKfVxQFRx99NAKBAL799lu88MILeP7553HTTTfFLKu6uhpnnHEGJk+enPJt6xGc3o5uARERERERERERUbfTZkHZlStXYtGiRXj66acxceJE7L///nj00Ufx2muvYevWrXFfU1NTg2eeeQYPPPAADj30UOy111547rnn8O233+K7774DAHzyySdYsWIFXn75ZYwfPx5HHnkkbr/9djz++OMIBAIRyzv33HNx6qmnYtKkSW21md2bg0FZIiIiIiIiIiKiVGuzoOyyZcuQnZ2NCRMmmNOmTJkCWZbx/fffx33NTz/9hGAwiClTppjTRo0ahYEDB2LZsmXmcseOHYvevXub80ydOhW1tbVYvny5Oe25557D+vXrcfPNN6d603qGEUcA2QM7uhVERERERERERETdjr2tFlxSUoKCgoLIldntyM3NRUlJScLXOJ1OZGdnR0zv3bu3+ZqSkpKIgKzxvPEcAKxduxbXXHMNvvrqK9jtyW2i3++H3+83H9fW1gIAVFWFqqpJLaOrUlUVmqZFbueMfxlPdkyjiLqguPsSEbUa9yWi1OC+RJQ63J+IUoP7EnV3rflutzooe8011+Duu+9udp6VK1e2drEpoygKTj31VNx6660YMWJE0q+76667cOutt8ZMLy8vh8/nS2UTOx1VVVFTUwNN0yDLbVpmmKhb475ElBrcl4hSg/sSUepwfyJKDe5L1N3V1dUlPW+rg7KXX345zjzzzGbnGTp0KAoLC1FWVhYxPRQKobKyEoWFhXFfV1hYiEAggOrq6ohs2dLSUvM1hYWF+OGHHyJeV1paaj5XV1eH//73v/jll19w4YUXAgjfibHb7fjkk09w6KGHxqwjEC9mAACiqElEQVT72muvxfz5883HtbW1GDBgAPLz85GZmdns9nZ1qqpCkiTk5+fzR5FoJ3BfIkoN7ktEqcF9iSh1uD8RpQb3Jeru3G530vO2Oiibn5+P/Pz8FuebNGkSqqur8dNPP2GvvfYCAHz22WdQVRUTJ06M+5q99toLDocDS5YswbRp0wAAq1evRlFRkTlY16RJk7BgwQKUlZWZ5REWL16MzMxMjBkzBg6HA7///nvEcp944gl89tln+Pe//40hQ4bEXbfL5YLL5YqZLstyj/ihkCSpx2wrUVvivkSUGtyXiFKD+xJR6nB/IkoN7kvUnbXme91mNWVHjx6NI444AnPnzsWTTz6JYDCICy+8EDNmzEDfvn0BAMXFxZg8eTJefPFF7L333sjKysKcOXMwf/585ObmIjMzExdddBEmTZqEffbZBwBw+OGHY8yYMTj99NNxzz33oKSkBDfccAMuuOACM6i62267RbSloKAAbrc7ZjoRERERERERERFRe2uzoCwAvPLKK7jwwgsxefJkyLKMadOm4ZFHHjGfDwaDWL16NRobG81pDz74oDmv3+/H1KlT8cQTT5jP22w2vP/++zjvvPMwadIkpKWlYdasWbjtttvaclOIiIiIiIiIiIiIUkLSNE3r6EZ0RrW1tcjKykJNTU2PqClrlINg9wGiHcd9iSg1uC8RpQb3JaLU4f5ElBrcl6i7a008kXsAERERERERERERUTtq0/IFXZmRQFxbW9vBLWl7qqqirq4Obrebd6qIdgL3JaLU4L5ElBrcl4hSh/sTUWpwX6LuzogjJlOYgEHZBOrq6gAAAwYM6OCWEBERERERERERUVdRV1eHrKysZudhTdkEVFXF1q1bkZGRAUmSOro5baq2thYDBgzA5s2bu339XKK2xH2JKDW4LxGlBvclotTh/kSUGtyXqLvTNA11dXXo27dvi9ngzJRNQJZl9O/fv6Ob0a4yMzP5o0iUAtyXiFKD+xJRanBfIkod7k9EqcF9ibqzljJkDSzgQURERERERERERNSOGJQlIiIiIiIiIiIiakcMyhJcLhduvvlmuFyujm4KUZfGfYkoNbgvEaUG9yWi1OH+RJQa3JeIwjjQFxEREREREREREVE7YqYsERERERERERERUTtiUJaIiIiIiIiIiIioHTEoS0RERERERERERNSOGJQlIiIiIiIiIiIiakcMyhIRERERERERERG1IwZliYiIiIiIiIiIiNoRg7JERERERERERERE7YhBWSIiIiIiIiIiIqJ2xKAsERERERERERERUTtiUJaIiIiIiIiIiIioHTEoS0RERERERERERNSOGJQlIiIiIiIiIiIiakcMyhIRERERERERERG1IwZliYiIiDq5W265BZIkRUwLhUK46qqrMGDAAMiyjBNOOAEAUF9fj7PPPhuFhYWQJAmXXnpp+ze4k9m8eTPcbje++eabiOkvvfQSRo0aBYfDgezs7FYt8/PPP4ckSfj888/NaWeeeSYGDx688w3ugeK9n21hxowZmD59epuug4iIiCgZ9o5uABEREVFP8vzzz2P27NnmY5fLhdzcXIwdOxZHH300Zs+ejYyMjBaX8+yzz+Lee+/FpZdeij333BMDBw4EANx55514/vnnceONN2LYsGEYPXp0m21LV3Hbbbdh4sSJ2G+//cxpq1atwplnnokjjjgC11xzDbxebwe2cOfceeedGDNmjBmYp8SuvvpqTJgwAb/99hvGjRvX0c0hIiKiHkzSNE3r6EYQERER9RRGUPa2227DkCFDEAwGUVJSgs8//xyLFy/GwIED8e6772L33Xc3XxMKhRAKheB2u81pM2bMwNdff40tW7ZELH+fffaB3W7H119/3W7b1JmVl5ejX79+eOGFF3DKKaeY05988kmcd955WLt2LXbZZZdWL/fzzz/HIYccgqVLl+Lggw8GIDJlP//8c2zcuDFFrU9Oeno6/vrXv+L5559v1/WmkqqqCAQCcDqdkOW27cw3ceJEjBw5Ei+++GKbroeIiIioOSxfQERERNQBjjzySJx22mmYPXs2rr32Wnz88cf49NNPUVZWhuOOOw5NTU3mvHa7PSIgCwBlZWVxu9wnmr6jVFWFz+dL2fLa28svvwy73Y5jjz02YnpZWRkApPS96goaGho6uglxybIMt9vd5gFZAJg+fTreeust1NfXt/m6iIiIiBJhUJaIiIiokzj00ENx4403YtOmTXj55ZfN6daashs3boQkSVi6dCmWL18OSZLMWpySJGHDhg344IMPzOlG1qbf78fNN9+MXXbZBS6XCwMGDMBVV10Fv98f0QZJknDhhRfilVdewa677gqXy4VFixYBAIqLi3HWWWehd+/ecLlc2HXXXfHss89GvN5oxxtvvIEFCxagf//+cLvdmDx5MtatWxezzd9//z2OOuoo5OTkIC0tDbvvvjsefvjhiHlWrVqFv/71r8jNzYXb7caECRPw7rvvJvWevv3225g4cSLS09PNaYMHD8bNN98MAMjPz4ckSbjlllvM7Tf+tho8eDDOPPPMpNaZSmvXrsW0adNQWFgIt9uN/v37Y8aMGaipqQEg2tvQ0IAXXnjB/MyNdhrfmxUrVuDUU09FTk4O9t9/f3PZL7/8Mvbaay94PB7k5uZixowZ2Lx5c8T6v/rqK5x88skYOHCg+b257LLLIm4aACJLOD09HUVFRTjmmGOQnp6Ofv364fHHHwcA/P777zj00EORlpaGQYMG4dVXX414fbyasgcffDB22203rFixAocccgi8Xi/69euHe+65J+Z92rRpE4477jikpaWhoKAAl112GT7++OO4dWoPO+wwNDQ0YPHixa36LIiIiIhSiTVliYiIiDqR008/Hddddx0++eQTzJ07N+b5/Px8vPTSS1iwYAHq6+tx1113AQBGjx6Nl156CZdddhn69++Pyy+/3JxfVVUcd9xx+PrrrzFv3jyMHj0av//+Ox588EGsWbMGb7/9dsQ6PvvsM7zxxhu48MIL0atXLwwePBilpaXYZ599zKBtfn4+PvroI8yZMwe1tbUxA4r97W9/gyzLuOKKK1BTU4N77rkHM2fOxPfff2/Os3jxYhxzzDHo06cPLrnkEhQWFmLlypV4//33cckllwAAli9fjv322w/9+vXDNddcg7S0NLzxxhs44YQT8Oabb+LEE09M+F4Gg0H8+OOPOO+88yKmP/TQQ3jxxRfxn//8B3//+9+Rnp4eUS6iswgEApg6dSr8fj8uuugiFBYWori4GO+//z6qq6uRlZWFl156CWeffTb23ntvzJs3DwAwbNiwiOWcfPLJGD58OO68804YlcsWLFiAG2+8EdOnT8fZZ5+N8vJyPProozjwwAPxyy+/mBnECxcuRGNjI8477zzk5eXhhx9+wKOPPootW7Zg4cKFEetRFAVHHnkkDjzwQNxzzz145ZVXcOGFFyItLQ3XX389Zs6ciZNOOglPPvkkzjjjDEyaNAlDhgxp9j2oqqrCEUccgZNOOgnTp0/Hv//9b1x99dUYO3YsjjzySAAi+/fQQw/Ftm3bzO/Rq6++iqVLl8Zd5pgxY+DxePDNN980+/0hIiIialMaEREREbWb5557TgOg/fjjjwnnycrK0vbYYw/z8c0336xFn7YddNBB2q677hrz2kGDBmlHH310xLSXXnpJk2VZ++qrryKmP/nkkxoA7ZtvvjGnAdBkWdaWL18eMe+cOXO0Pn36aBUVFRHTZ8yYoWVlZWmNjY2apmna0qVLNQDa6NGjNb/fb8738MMPawC033//XdM0TQuFQtqQIUO0QYMGaVVVVRHLVFXV/Hvy5Mna2LFjNZ/PF/H8vvvuqw0fPjxm+63WrVunAdAeffTRmOeM97S8vDxiOgDt5ptvjpl/0KBB2qxZs8zHxnYuXbrUnDZr1ixt0KBBzbapNX755RcNgLZw4cJm50tLS4tom8HYxlNOOSVi+saNGzWbzaYtWLAgYvrvv/+u2e32iOnG52p11113aZIkaZs2bTKnzZo1SwOg3Xnnnea0qqoqzePxaJIkaa+99po5fdWqVTHvc7z386CDDtIAaC+++KI5ze/3a4WFhdq0adPMaffff78GQHv77bfNaU1NTdqoUaNilmkYMWKEduSRR8ZMJyIiImovLF9ARERE1Mmkp6ejrq4uZctbuHAhRo8ejVGjRqGiosL8d+ihhwJATEbhQQcdhDFjxpiPNU3Dm2++iWOPPRaapkUsY+rUqaipqcHPP/8csYzZs2fD6XSajw844AAAwPr16wEAv/zyCzZs2IBLL700pq6rUaqhsrISn332GaZPn466ujpzndu3b8fUqVOxdu1aFBcXJ9zu7du3AwBycnJa83Z1GllZWQCAjz/+GI2NjTu8nHPPPTfi8VtvvQVVVTF9+vSIz7KwsBDDhw+P+D54PB7z74aGBlRUVGDfffeFpmn45ZdfYtZ19tlnm39nZ2dj5MiRSEtLw/Tp083pI0eORHZ2tvldaE56ejpOO+0087HT6cTee+8d8dpFixahX79+OO6448xpbrc7bqa5IScnBxUVFS2un4iIiKitsHwBERERUSdTX1+PgoKClC1v7dq1WLlyJfLz8+M+bwx6ZYjuUl5eXo7q6mo89dRTeOqpp5JaxsCBAyMeG4HRqqoqAMCff/4JANhtt90StnvdunXQNA033ngjbrzxxoTr7devX8JlADC77HcGNTU1EfVYnU4ncnNz4847ZMgQzJ8/Hw888ABeeeUVHHDAATjuuONw2mmnmQHbZER/nmvXroWmaRg+fHjc+R0Oh/l3UVERbrrpJrz77rvmZ2fdFiu32x3zHcvKykL//v3NQLt1evTy4on32pycHPzvf/8zH2/atAnDhg2LmW+XXXZJuFxN02LmJyIiImpPDMoSERERdSJbtmxBTU1NswGl1lJVFWPHjsUDDzwQ9/kBAwZEPLZmRxqvB4DTTjsNs2bNiruM6JqsNpst7nytCZAa673iiiswderUuPM09z7l5eUBQFLBv5YoirLTywCASy65BC+88IL5+KCDDooZiMrq/vvvx5lnnol33nkHn3zyCS6++GLcdddd+O6779C/f/+k1hnv85QkCR999FHcz8kYFE1RFBx22GGorKzE1VdfjVGjRiEtLQ3FxcU488wzzc/HkOgz35nvQiq+R/FUVVUlDEoTERERtQcGZYmIiIg6kZdeegkAEgYhd8SwYcPw22+/YfLkyTuUHZifn4+MjAwoioIpU6akrE0A8McffyRc5tChQwGIzM0dWe/AgQPh8XiwYcOGpF+Tk5OD6urqiGmBQADbtm1r9frjueqqqyK64ydTWmHs2LEYO3YsbrjhBnz77bfYb7/98OSTT+KOO+4AgFZ/psOGDYOmaRgyZAhGjBiRcL7ff/8da9aswQsvvIAzzjjDnL548eJWra+tDRo0CCtWrIjJfl23bl3c+UOhEDZv3hxR7oCIiIiovbGmLBEREVEn8dlnn+H222/HkCFDMHPmzJQtd/r06SguLsY///nPmOeamprQ0NDQ7OttNhumTZuGN998E3/88UfM8+Xl5a1u05577okhQ4bgoYceigmCGlmQBQUFOPjgg/GPf/wjblC0pfU6HA5MmDAB//3vf5Nu17Bhw/Dll19GTHvqqadSlik7ZswYTJkyxfy31157JZy3trYWoVAoYtrYsWMhyzL8fr85LS0tLeY9bM5JJ50Em82GW2+9NSbjVNM0sxavkaVqnUfTNDz88MNJr6s9TJ06FcXFxXj33XfNaT6fL+73HQBWrFgBn8+Hfffdt72aSERERBSDmbJEREREHeCjjz7CqlWrEAqFUFpais8++wyLFy/GoEGD8O6778LtdqdsXaeffjreeOMNnHvuuVi6dCn2228/KIqCVatW4Y033sDHH3+MCRMmNLuMv/3tb1i6dCkmTpyIuXPnYsyYMaisrMTPP/+MTz/9FJWVla1qkyzL+Pvf/45jjz0W48ePx+zZs9GnTx+sWrUKy5cvx8cffwwAePzxx7H//vtj7NixmDt3LoYOHYrS0lIsW7YMW7ZswW+//dbseo4//nhcf/31qK2tRWZmZovtOvvss3Huuedi2rRpOOyww/Dbb7/h448/Rq9evVq1fanw2Wef4cILL8TJJ5+MESNGIBQK4aWXXjKD5Ia99toLn376KR544AH07dsXQ4YMwcSJExMud9iwYbjjjjtw7bXXYuPGjTjhhBOQkZGBDRs24D//+Q/mzZuHK664AqNGjcKwYcNwxRVXoLi4GJmZmXjzzTdTUg4ilc455xw89thjOOWUU3DJJZegT58+eOWVV8x9KDqTePHixfB6vTjssMM6orlEREREABiUJSIiIuoQN910E4DwQE9jx47FQw89hNmzZyMjIyOl65JlGW+//TYefPBBvPjii/jPf/4Dr9eLoUOH4pJLLmm2C7uhd+/e+OGHH3DbbbfhrbfewhNPPIG8vDzsuuuuuPvuu3eoXVOnTsXSpUtx66234v7774eqqhg2bBjmzp1rzjNmzBj897//xa233ornn38e27dvR0FBAfbYYw/zPWzO6aefjmuuuQbvvvtuRNmARObOnYsNGzbgmWeewaJFi3DAAQdg8eLFmDx58g5t484YN24cpk6divfeew/FxcXwer0YN24cPvroI+yzzz7mfA888ADmzZuHG264AU1NTZg1a1azQVkAuOaaazBixAg8+OCDuPXWWwGI2sKHH3642a3f4XDgvffeM+vYut1unHjiibjwwgsxbty4ttvwVkpPT8dnn32Giy66CA8//DDS09NxxhlnYN9998W0adNibnAsXLgQJ510Usr3MyIiIqLWkLTONBwtEREREVGKzZkzB2vWrMFXX33V0U2hdvTQQw/hsssuw5YtW9CvXz8AwK+//oo999wTP//8M8aPH9+xDSQiIqIejUFZIiIiIurWioqKMGLECCxZsgT77bdfRzeH2kBTUxM8Ho/52OfzYY899oCiKFizZo05fcaMGVBVFW+88UZHNJOIiIjIxKAsERERERF1aUceeSQGDhyI8ePHo6amBi+//DKWL1+OV155BaeeempHN4+IiIgoBmvKEhERERFRlzZ16lQ8/fTTeOWVV6AoCsaMGYPXXnsN//d//9fRTSMiIiKKi5myRERERERERERERO1I7ugGEBEREREREREREfUkDMoSERERERERERERtSPWlE1AVVVs3boVGRkZkCSpo5tDREREREREREREnZimaairq0Pfvn0hy83nwjIom8DWrVsxYMCAjm4GERERERERERERdSGbN29G//79m52HQdkEMjIyAIg3MTMzs4Nb07ZUVUV5eTny8/NbjOITUWLcl4hSg/sSUWpwXyJKHe5PRKnBfYm6u9raWgwYMMCMKzaHQdkEjJIFmZmZPSIo6/P5kJmZyR9Fop3AfYkoNbgvEaUG9yWi1OH+RJQa3Jeop0imFCr3ACIiIiIiIiIiIqJ2xKAsERERERERERERUTtiUJaIiIiIiIiIiIioHTEoS0RERERERERERNSOGJQlIiIiIiIiIiIiakcMyhIRERERERERERG1IwZliYiIiIgoKf7167HxlFM7uhlEREREXR6DskRERERElJRQWTmafvmlo5tBRERE1OUxKEtERERERMmRpI5uAREREVG3wKAsERERERElhzFZIiIiopRgUJaIiIiIiJIiybx8ICIiIkoFnlUREREREVFyGJQlIiIiSgmeVRERERERUXJYU5aIiIgoJRiUJSIiIiKipEgMyhIRERGlBIOyRERERESUHAZliYiIiFKCQVkiIiIiIkoOa8oSERERpQTPqoiIiIiIKDkSLx+IiIiIUoFnVURERERElBRJZvkCIiIiolRgUJaIiIiIiJLDmrJEREREKcGgLBERERERJYc1ZYmIiIhSgmdVRERERESUHGbKEhEREaUEg7JERERERJQUiUFZIiIiopRgUJaIiIiIiIiIiIioHTEoS0RERERESdE0raObQERERNQtMChLRERERETJYUyWiIiIKCUYlCUiIiIiIiIiIiJqRwzKEhERERFRkpgqS0RERJQKDMoSEREREe2kQFERAkVFHd0MIiIiIuoi7B3dACIiIiKirq7swQehVFVj0PPPdXRT2hYH+iIiIiJKCWbKEhERERHtJFt2NpTq6o5uRttjUJaIiIgoJdolKPv4449j8ODBcLvdmDhxIn744Ydm51+4cCFGjRoFt9uNsWPH4sMPP4x4XtM03HTTTejTpw88Hg+mTJmCtWvXRsyzYMEC7LvvvvB6vcjOzk71JhERERERRZKkjm4BEREREXURbR6Uff311zF//nzcfPPN+PnnnzFu3DhMnToVZWVlcef/9ttvccopp2DOnDn45ZdfcMIJJ+CEE07AH3/8Yc5zzz334JFHHsGTTz6J77//HmlpaZg6dSp8Pp85TyAQwMknn4zzzjuvrTeRiIiIiHo6RYUk94BOaMyUJSIiIkqJNj9zfOCBBzB37lzMnj0bY8aMwZNPPgmv14tnn3027vwPP/wwjjjiCFx55ZUYPXo0br/9duy555547LHHAIgs2Yceegg33HADjj/+eOy+++548cUXsXXrVrz99tvmcm699VZcdtllGDt2bFtvIhERERH1cJqqwLd8OYIlJR3dFCIiIiLqAto0KBsIBPDTTz9hypQp4RXKMqZMmYJly5bFfc2yZcsi5geAqVOnmvNv2LABJSUlEfNkZWVh4sSJCZdJRERERNSmVJFBqgWDHdyQtqUxU5aIiIgoJextufCKigooioLevXtHTO/duzdWrVoV9zUlJSVx5y/Rsw6M/5ubZ0f4/X74/X7zcW1tLQBAVVWoqrrDy+0KVFWFpmndfjuJ2hr3JaLU4L5EXZGmhAAAaiDQab67bbEvGUHZzrKNRO2Fxyai1OC+RN1da77bbRqU7Uruuusu3HrrrTHTy8vLI2rVdkeqqqKmpgaapkHuCbXQiNoI9yWi1OC+RF2Rr6EBALC9tAy2tLQObo3QFvtSaPt2AEg4PgRRd8VjE1FqcF+i7q6uri7peds0KNurVy/YbDaUlpZGTC8tLUVhYWHc1xQWFjY7v/F/aWkp+vTpEzHP+PHjd7it1157LebPn28+rq2txYABA5Cfn4/MzMwdXm5XoKoqJElCfn4+fxSJdgL3JaLU4L5EXVGxLCMAICczA+6CgojnQuXlsOfnt3ub2mJf8pWVoQ5AQdQ2EnV3PDYRpQb3Jeru3G530vO2aVDW6XRir732wpIlS3DCCScAEDvgkiVLcOGFF8Z9zaRJk7BkyRJceuml5rTFixdj0qRJAIAhQ4agsLAQS5YsMYOwtbW1+P7773HeeeftcFtdLhdcLlfMdFmWe8QPhSRJPWZbidoS9yWi1OC+RF1OIIi8s+dAUpSY7+2fBx2M0atWdkizUr0vSZJYDvdN6ol4bKKurOmP5fDstmtHNwMA9yXq3lrzvW7z8gXz58/HrFmzMGHCBOy999546KGH0NDQgNmzZwMAzjjjDPTr1w933XUXAOCSSy7BQQcdhPvvvx9HH300XnvtNfz3v//FU089BUDsvJdeeinuuOMODB8+HEOGDMGNN96Ivn37moFfACgqKkJlZSWKioqgKAp+/fVXAMAuu+yC9PT0tt5sIiIiIuphbNnZ0EKhjm5GG+NAX0REXdHGv/61w24QElF8bR6U/b//+z+Ul5fjpptuQklJCcaPH49FixaZA3UVFRVFRJH33XdfvPrqq7jhhhtw3XXXYfjw4Xj77bex2267mfNcddVVaGhowLx581BdXY39998fixYtikgRvummm/DCCy+Yj/fYYw8AwNKlS3HwwQe38VYTERERUY8iSYDdDi0Y7OiWEBERRdAUpaObQERxSJoxhCpFqK2tRVZWFmpqanpETdmysjIUFBSw+wDRTuC+RJQa3JeoKyi57Xb0vuF6SPp3dPN558P7l7+g7J57YjKRVo4ajVErlpvztpe22Jea/vc/bJz+f8y2oh6HxybqylSfD6vH79Epfrs7676kNjQguHUrXMOHd3RTqItrTTyx8+wBRERERERdRNWrryK4dVvENLU+8Wi7ms/X1k1qH8znICLqcrRAoKOb0OnVfvQR1h97XEc3g3oYBmWJiIiIiJKg1NWhdvFi87EWjLzITT/wwPBzevDSqDGrNjW1Qws7h/XHHtvRTSAiIoueVlrnzyOPQqCoqHUvkqS2aQxRMxiUJSIiIiJKgu+PP1B80cXhCVGDennGj0f6wQej6fc/sP4YEZg0grHdJiibRKasf+26dmgIERElq6cFZQMbNiC4dRvUhoZW1NNlUJbaH4OyRERERERJ0EKRF3ZxL/TsNvhWrUTgzz8BAGptrfi/sbHV61MbG6FUV7f6dURERFY9sXxBcOtW/HnU0aj98KPkXsBMWeoADMoSERERESVBUyIzY7VgKGYe2eNF9RsLzceKHpTVdiBTtvzRx1A075xWv64tcYxgIqKup6cFZd1jx6Lxxx8RKi3tcVnC1LUwKEtERERElIyocgXQg7SVL76EwIYNAERmrO/3381ZgttK4Bw0CGpT6wf60oLBHQrmtinGZImIupya99/v6Ca0m5r3P4Bks5k9TSSnM7kXMlOWOgCDskRERERESdCigrLG49K77kJg40Yx0WEX/8symn79FYGiTXCNGQ21qRFqQ0PrVihJYBSUiIh21vYn/wGgZ/R22HrFFWiy3ByFqqD48itafiFjstQBGJQlIiIiIkpCdE3Zxp9+Fn/Y7THzSi4Xyh95FGp9AxwFBdCamrB6rwnm88HSsjZta9vp/hf0RETdVk/pyq+qAAA5IwPBklLUfvBBjwhIU9fDoCwRERERUZTyRx6JeBzYsgW1H4nBQipffVXM8+CDCFVVQbLZwjPq13yy0wmlrg4Vjz0Ge0EBVL0MgarX9Vt30EFQamqab4TUdbOaumq7iYi6MzXQvYOyoe3bxR/6Mcg5eDCCmzcDANSGFgbcjC5RRNQOGJQlIiIiIorS+MOPEY9r3nkH9UuWAABKb7vdnK75/ZAcjvCMenaOLTcXal0dAMDRty/U+noAwLYbbjBnbSkoK0lSUompqs+Hhh9+aHnGVEg22KppUKqrsWrPvdq2PURElLRtN96AYElJRzejzaw7+BAAEMdlTYPsdkP1i5rual1ts68NbN4Ce98+bd5GIisGZYmIiIiIILJjqxYuBAAoehDVYA64JUeePmuhEJwDBlgmiKClY+AAKA31SDvwANh790aoqgoAECzabM5qDEKSmJRUEDRQVISiM2a1OF9KtNAeTQ9KQ1EQKCqC1thCZhIREbULyeVC3UeLzMBld6Tp5Rmcu+wCLRSCnJ6Ohm+Xief0niqJKNXVsGfntHkbiawYlCUiIiIiAtDw7TI0/fIrAJiZrYaaDz4EAMhud8R0LRCELcdyEadpSD/0UEiyTXSVVEWmTtWr/wIAKHpwFhAZrs2SWg7KKvX1UFsqg9CeFFF3N1RRgWBpadxZmn791SwFQURE7UNOT+/oJrQb2eOB5vNBTk+HUlEBIHawzmhaIADJ6WyP5hGZGJQlIiIioh5PU1UENm40yw8YpQcMoW3bAACSxxP5ulAQkssVfgxNBFMBkSWqKpA8Hqi1otukaskc3Xbd9fCtXJm4UUkEZSseexybzprT/MalUpKZsluvuRbbrr0u7jzVb7+N7f98OuVNIyKixOT0tIjH/vUbOqglbU92u6EGApDdluNz1GCd0bRgkEFZancMyhIRERFRj6dUVkKprkbN229D07SI8gWaEr6Qi8mU1btKjl6lB1c1DZAsz6saZEvQ1tG3r/l3cMsW+NesSdyoJIKykOXONZq2/l5JDgfUhoa4s6gNjZC8nrjPERFRahkDL8ppkUHZ9Ucd1RHNaVPeSfsAALJOPBGazwfJFT5ma6HYY6VZcgd6pqy1RjxRO2BQloiIiIh6PGspAS0QABQFSnU1tFDIDLwCgOz1wL3rrujzt7vEhFAoInCq1tbBlp4Rzp5VVdjz8wEA2Sf/FU2//RaxXiNzR6mrixl8RZIl82I6Eclub92G7qSW2mNc4BqDrMSlqpBkW6qbRkRE8eg3y2xp3b98gawHYTOPOhKq3wfZ7cKAfzwpnoxTvmDVmF3N+u5aMAjJ4TCPc0XnnBNzzCZKNQZliYiIiKjHUxtEWYH0gw8268mu2W9/1C76OCIoK3m80DQVWccfj/xLL4l4DgDSDzoQGVMPhy1DXPxqqgLJ4YB7zBjAFhuI1BRxkVjxxN9RdObsyCfjZMpuf/75iAHC2jsoi5bGHbNkysZT8957kJzOmPeNiIh2TshSszyCfrPMWlO2pRtsXZ4sQ/P5ITld5k3SrVddHXeAzej3Ys3EfRAsKUHDF1+i7N772qO11IMxKEtEREREXZ6vuTIASdCaGpF/6SWwZWVCqawUExUFstsFLRiEd++9AejlC4JBSJIEye0WA4dI4XoFvc47DxmHHALZ6xUTVHGxp+nRzPRDDhHZpPprSm66GZUvvoi6z5ZE1LKr/+YbVP/7TUhRgdyyv92NwKZN5mPJ0c5B2RaYpR7k2MsMTVWx9cqrINnt0AIB+P/8s51bR0TUfa2dtG/cARaNHgyypSa6ptc3L/3b3e3TuPaiH1slSYLm90Nyu2HUFAps2hR/AEpLiSItFIJaW4tAUZF4TXFxmzeZejYGZYmIiIioy9tw3PE79XqloQGy1wvJ40GoMpxtVPHE3xHcug3OIUMARF7Uyl5vwrqpkhGUlcXFoLP/ANh75UNyudDw9dew9+4dXsfjTyC4qSgi67XyhRegVFXFDbpGZJm2e6ZsC9lVSjMDqRjP2WQEioqw/uhjUtcuIiKC1tQU8ThYXBy3B4Oqz1f5/PPt1rb2pvpE+QKjRwqglyeCyI5dOWo0AKBh2XeoXfQxAMA7UdyA1fSSRqFt28zXELUFBmWJiIiIqEvT9Dpxqt+/w8tQa2pgy86G7E2DUlVpTvetWIHg5iLzYlbyeiC5RWDWOXAQAkWb4y7Ppg+oYs/JAQD0e/AB9Dr3HADA5nnnwDV0qDmvUlMj/rAEYBUjMGyPLQNgBGWVujrUvvtuq7e1LVkHTUn4nAZoO/FZERFRfEpdfcTjdZOnmN3zrb0xzBuKsoz6L77oliVlNJ8Pclp6RC1ZMyhrCV4HijYhVFYKaBo8u+0GAFCbwnXm1ahAN1EqMShLRERERF2aUQM2Xq24ZCk1NbBlZUHz+bD9qX9GLr+xyQzKym6PKGEAMZK12tQYd3mZxx6LgquvRuFttwEAJJtNdNs3LnwtJQ8Mki0clNX8PvN10YxlNP38M/xr17VmM1NgJzJlo0a5JiKi1Np48snm36v3nij+0H+XZW+4p4eqly+AzYbN55yL8sceb7c2trXh33wNAJA8HsiZGXCNHAnn4MEAwjcElfpw8FrZXmne1DXq7mq+cCCWQVlqSwzKEhEREVGXZlxcBnei9ptSXw85PR2B4i3wrVgRs3wzKOvxQPKIoKzkdCTMLnL07o282WeambIGIxhp1py1sgRqNb0WbbyBvIx1WjN5AKBh2bKE25cyLZQviMmUtQSVNUV/LjYeTUREKbJqjz0BAGptLQDxu5x59NHw7rOPOY9x3DRu/CmV29u5la2nhUJJZfTa8/IAAMMWLUL6gQfCUViIPL2nihF8VevDpYeqXn0Vmj8ASJKZTWwGrSFuzBK1FQZliYiIiKhL04JB2Pv2QWBT0Y4vwx+A5HTBliayZGS9/ACgB2X14KjkcUPWyxdIdjvQyi6fRlDWWtvPXE9DA+q/+EI8MAKX9viZsv71G7D96acjphfNPqtVbWkTUZmyEUFlVX+umw/6TUTUkbSmJrNkAQCoDY2QPG5kHHII0g46ELbsbLN8gRHklBzOuMvqTOqWLkXNBx8kPb+jdwFklwsAIOmlgDS/OAZH14M3e6foQdnAli3mc4l6xBClAoOyRERERNSlacEgbOkZO9UlXvPrA4LomZ7WbBy1qRGS01K+wMiUdSTOlG2urQDMbFsr/6pV2HzOueJ5/QJSralFw7ffRmTtaMEgQuXl8K1a1ap1p0RLmbJGUFZ/H42gbNHceVDr6tq0aUREpLPWUQ0GIEki9DPg739H3tyzw0HJOIOAdVaazwfN54emqqj9+JNWvdY4FmlBIygbWXtXqakFNM0M4ioVFeH1snwBtSEGZYmIiIioS9OCQcjp6Ts8eFTx/PnY/s+nIbnd6HXeeWKZgQBcw3cRfzeFa8rasrLMgb52JCibMflQFFx1FfIvuST+DMaAYvoFpH/tWhSdNQdNf/xhzqIFAiJzx3LR3WnoF/ianhVrbEfDV1/Bv359hzWLiKi7Sz/kEOTOOgMAoPrDNym1YBCwidCPJMuQXO6IG31A5CBgnU39V1/Bv3YttEAQJbfcgsrnnkdxomNoApLe60QzyxdEBmWr33gDDd9+C0kPyqqNjbDn50NyOCJqypY/8ujObEqPoQUCEXV7KTEGZYmIiIioQ/jXb4B/w4adXo4IyqZBC+xYUNa3ajUAQHK54B45wpwup2cACNeUteXmQk5LMwf6khwOVL36r1YFZvPmzEHeWbNhz88HALjHjo07X/QFsmSzQdM0UaZhw4YOy9zRkqwpa74nluwrpapKPKd0wmAyEVE3IGdmAgDqFn1kTtOCQUhyOPQju11QG8JB2ZxTTzWDkZ1R6Z13ofrNt8zjilJVGXe+Zo9P+g1Cs6ZsY2xJAi0QAPT3SQspcO+2Gwpvvy1c6iEQQMUTT+zwdnSkVJxrtcb2555H0Rmz2nWdXRWDskRERETUIUoXLED5gw/t9HJE+YJ082Kr1a83Bt+yXJTKaWlw7bILet94g6jH53BgxLffQHI6wgN96QFHI9jYGpIkofCWmzH49dcipxv/22zmxSEgMmYRDMJR2AdKTU2HDDyy7ZZbWi4RYWTKGrVzLTVlQ+XlEc8REVFqGbVht//TUnM8FALkcH1yyeVCYMtm87Fnj/Gd+maZ5HJBC/jNY4c5aGQ0VY04bkYsw6gp6xPnCVowzvY6HLBlZ2Po+++JQLbDATktzQzKhqqrd25D4giWlqZ8mdHWTNoX6488qs3XY6X5fQjtwLlRT8SgLBERERGlVHDr1qTms+Xk7FBAM9qmWWdCcjjNATxazah/qmfAZp8yA46+fSE57HAUFoqMGmOgL4cDsscr/tazWRV9hOvWypkxIyJ7yUpyOMxRsQGg5JZbofp8sGVkQPX7ofraPyhb/drr0Hy+ZucxLpa1gDF4jAOaqsLetw+afvufeM4XDp5rgQC0zliGgYioCzKOG5plREWRASqZj5t+/Q1VL76ErOOPBwDYcnOBUOQgjZ3FxhmnIFRSAtXvNzNljWOGGn2DT1ESH1ONkkB//imWoZfYcY0ahbQDDxAzBYOQJAmuXXaBWl8PyeUSN3z1oKxRF71o7rwd3h5j/YAYbGzdQQfv8LKS4VuzJiXnWa1V8cTfI+ryUmIMyhIRERFRSq07dHJS80k2GZqmYvuzz+3cCkMhqAH/DteUNQOuepC1z803A9AAu92sJ2dkxYqgrAjeyh5RW1ZJcfaMpqpifQ4H+j/+mDk9WFwMOTMTWiCYsHxBS+UFWitQVBSZ2SpJiWcGAP1C17h4lt1uIBSCLTMLqj66tfE/AGy+6CJUPPH3lLaZiKjHMoKSqiUoGwxCsmTKZkw9HIC4MQoAtvT08CCNnUzTr79Cqa6G5g+Ey+LoWb3RNwk1VQUsNzOjnhQvra3RlyG2N/2Qg5ExZUrs/DYbIIleM0ZtVONY2PDVVzu8PeuPPibcpHZ4z63nRcXz57f5+iLWHQx2zICkXQyDskRERESUcubIzs2RZEADyu65Z6e6uXn23BO9zjkH6g7WlDUyiyRLwHHA088g/6KLRFBWrykLAK5hw+AZN86cr8+CO6DuYKZs/MZIUOvrIWdlQna54Bw8WGQxQXT/t2Wki4G+GpvMYLLBlpUFtaZmh1dd/sQTMYNx/Xn4VNR9/nl4Qks1ZRUFsNvNi2fJ44EWCkGy281MZmumbMMXX0LRs4+IiCg1rME4LRSK6NbvHrMrXMOHmzciJY/HDHR2JhH12lUlXL5Az+qNCWqqKiQ5/o1DpVYcZ+o/XYJtN95ovjb/gguQM316zPyS3Q6EFMiWTNkdLb2zctToyLboN3KNjN+GZct2aLnJsA5oVrfkszZbTzTvpH0AAI3//and1tlVMShLRERERCmj1IuLl2QCbUawDgh3C9wRtqws2Hv33qHyBaI+a+yAH47eBbBlZEByOBCqqIAtMwsA4Bw8OCIoa8vO3uF2J2xTdTVs2dmQPG64hg3DoJdfBgBsnncOGn/8L6BpUJuaIHu9Ea9zDh0KtYXyAs2peORRs8RABEuwOmEtP7PxiihZYFw8NzVhzT6TxDQ9SBCd3ZToIpqIiHaMtca6FgxCsoVDP5LDDtXvN4+/ssdjBjo7k/pvvol4HF2+wCg9ZD6vqhG1c60chb2RdfxxAIDqhf8GFAV97rzTfA88e+wRMb9ks0ELhURN2fqdC8oajM8ksHlLxHaEysp2arnNrrOhAY7+/SPW1x7MGr6h5AdC7akYlCUiIiKilFkzYQIAUQO1JZrfD8klMnUiMmJ2gOxy7VD5gm033IhQMwNtSA4HQqWlsOXmxF9vWnqr19ksmw1KdTXsOTlm7VpbZob5tNrYKIKyDQ1wDRliTi+8/Ta4x4zZ+QtrtYXXt/C8pqgiK9byeWqBAHx//GF+PjEDskVl/BIR0Q7S73FZb35pwWDkQF92u3jervcScTo75UBf1vqwmqaFg7L6tkVnyhbPn58w8OgZNw597747vLyQAkf/fubj3FlnRL7AboOmKJDTLDVljRu/es+ZZFhLClW+8KKYFtSXEwpBzsho02OgUl+Pgisu1x/oGcaqKgLYbcisbbyT53Y9AYOyRERERBTX9meehZqgdmk81pP8emuX90TzKwokm7gY2dkMDsnphLYj5QvsCerPmc/bAVWFLSMj7tNG98+dkXfeubD36SOWpwdlbdnZoh4rIC7ajPXp05SqKgx84fnwdFkWGT87eWHdUo27Fi/kVEVc8AcCcPTtC+iDzWjBoBmMNYPnNhvk9PSE7y0REe0YLSpT1jrQl2SzRdSZlWw2M2DXqVjrw2rhAF/Dd9+JaVHHo4Yvv0q6nJAWCJhZsjHrgp7pGQpB9nrM3jRGpqzUmiCq3uZQVZVZSkAL6jVxFQW27Gz4/lget8dOKqgNDZDT0sQDfRtL71iA0rv+1ibrM+k9bIyyUP7161H70Udtu84uikFZIiIiIoqr7N57I7rVaYEASu+5N+H8RmDVO2ECMg4/PLmV6CfsxkVKawQ2bw4vxm5vuWu9Rd2SJSi+/ArIThcAIOeM0+M3zxjgy+WO/3wKgrLZ0/4KW5Yoj2ANykrGgGIuF7JPmQEA8Iwfh/qvvkLdkiVm0Fa8UBaZPTsT3LbZYi5ygchau2jmPVZqa+HfsEF8FqEQhi35FFogaGYBmdlNRhs1TdTra0w+8E9ERMkpuPIKAOL4KkV169dUFZJxU9Jm65TlC4wu8AAATTODokplpZgW53gV0xMjgfKHHorIxPXutRf63ntPeN16+QJJls1a6lvOPx+u4bvANXx4UusIbd8OVW/z2kn7oub99wAAtYtEcFILhmDLykLlc8+hdtHHSS2ztdR6S1BWUVDz7rsIFG9BcNvWNlkfoJfLMALX+vlDwzffoviy+Wj8+Zc2W29XxaAsERERESVm6XoXLCtH5bPPJp5XD7bZ+/RJrvaappnLN7vzJd0sDX8elmTgNw61vh61H3wAW7YIhhZed13c+YygrOyKH3xNNL01JIfdLOMAmw2BjZvg6N8fsttjzpN9wgkAgL4LFoiMpqiLUckmQ7I7durCWpLlJDJlEz9f9corKL39DvGeqSokSYKmaXD2E11Eza6nlu6Mcnoatj/11A63mYiI4jNqnocqKiIyZcXEEKD3VJHsdtT85z+o/+qrdm5hC6wDS1rKFwAAZDluz41W1TC1hTNe7bm5yDr2WPOx5LDHPR72Ou882PXBN5sTqqzE2v32jzgXCm3dhuyTT0b1v14TE5SQ5YZs24TmRKZsuMxSYFNRm6zHSqmvh6yXXTI+M+O7WPPOO22+/q6GQVkiIiIiSshaD02p3N78vHpQVnImX2/N1NoMTyPAp6oRo0onTb8Yc/Tti4HPJQ40Sw59dGqXq9nl7AzJbjczdpXKSjR8+y0chYUouPLK8DweT6KXo+Dqq+GdMEHvgroTmbKy3GwmrGhg4ueNbGcjKCsmamaWjqYokNzuiExZm36xaNTsIyKinaBaApl6dmz5Aw+YNT4NonyQDSO+/87MGPWvXdfsoq3nA+1BCwbhHDTIWHk4wNerlzjGxLuJ2MKNyYzDDzeX2Vwg1D12d3jGjo2ZLjmdqP/8cxTPn9/sepSqKvF/TU3UAvTeQZoGLRQOyhqfVf0XX6DsoYeaXXZrRJQvAKA2NaHhiy9Ttvy466ypgS0jU/xtZC7rNwWkVtTj7SkYlCUiIiKixCyZKKpl4JB4jKySpAd2sHSLb223ezWgB2WbmiK78SdLD15qoVCz9eHC5QsSBGWl+JNbQ7LZIpYf2LgRkscL98gR5jTZmxbvpQCA7JNPhqNfP7N8QUi/GGw1m63ZTFgAgNZcUFb/3B328MW7qprZyFooJD4ra6asfrHYmtrFREQUn9oUrk0aEXSUIkM/WigE2GQRFNSPgWp9XcLlKrW1WDV6DCqefDK1DW6GFgrCu/dfxANJMrNO5TQxCGa8kkUtnUv0f+Rh9Flwh3jQzE3VvNlnouDycODVOL4Zx+qG739odj1GiYX1Rx4VMd0Iwirbt2PzuedBzsrUV6AiWFKCzeeci5q3U5dNqtbXw5YePn+oee9dAICz/4CUrSOaUlcHOUPccPUtXw4A0HwiOMugbCwGZYmIiIgoIWuAtaVgq/G8UlEREXBN/ALNzHJt7Qi9RrkD1ecza682/fwzfKtXJ/V6o86bFlKaHflYcujdOxNk48peb+KAbbLsjohlaIoC2R25TNkTP/A8+I3XzQtUo3zB2kn7QklysBMrCTDG5Uqoubq95kWrwxnOYNI0uMfujuHLvgWCQUheT/iz1rNo3bvtlly5CyIiapZmvcFlCcTGZIWGQuZAm0YWrdqQeLApY+Cw8oceRmh74l4zmqa1PCBkkrRgEG4jW1WWoYVCcA0fjr53/Q0Zh00B4txEzJgyJYkFiwNda0oGGDcOzRu1tthBQuu//BJaKARNVbHp9DNgy+8FAPDsuaf4f9w489xo7f4HIFRaClu6CF7WLvoY66YclnR7km53o8iU7f/EEwAg6rwj9pyr7OGHUfGP1JQSCm3fDnteHgAx+BoAqD79/WvNIGk9BIOyRERERJRQRFC2pcCZnqESKNrc/HwWst4tv9VBWb0talOTWXtVqa5G1csvJ/d6fwCSywUtFEwqUzYRR+/eGPXbr8k1OgHZ7ULG5MkYtXIF7L17i0BzVJtkjycicGsMNOLZfXdzIC5r+YJgSUmr26EBkKLrDgKRF9jNZNKa5SvsdvM1mqZBssmw5+SIYLPXawbEAQB2G7x/+UvSg7MQEVFi9r59w4FM/dhgy+8VkykrntCn6QHG5rJMrccBc6CtOCqffwFbzju/la1OsM5g0DxHMHqlDH3vXXj33APOIUMjbhJqqoq0/fdH/kUXtrxco8RDnMBqXDYZfx51tHitUR7BHvvazfPOgW/Vaqh1IuPYPWYMAMA7cW+xCU5nzLmOc/AQAEBw61bzfEOpqcHKUaMT3mRuzU1XLRiC5HAg49BD0Pv666EFgyi87VbUvv8+lPoGaIoC//r1CPy5HjVvvQUAaFq+HA3ffpv8OhQFW6+9Dv4NG8Trf/oJ9oIC9LnjdqTtt59Yjx7Ubu25Xk/AoCwRERFRD7f++BPgW7ky7nORQdkWMmUVBRlHHIG+d90JAGj69deE86p+PySHw7zgUuvrW9VmIyj755TDwhdtACRXcqUMtEAAklNkdO5MUDYVJLsd2SedCEmSkH/ZpRHd+w2y1xsR/C289ZY4y7Gh+k1xUaXtSDkATYvIcG767Tfxh+VCvdkMKP25iJqyqgrjaloLhWDPzolYniTbxIVqC98tIiJqWe6pp4a75+s/50p5RWxpGlk2ezQYN/aa7fpvqd/a3ICQZXff3eyxv1VCofAxOKREDvxlk6GFgmZbql9/HQ1ff53UYh364JP2Xr2Sm7+gt+gBhHAvHSlB6QO1oQEhPWjtHjUaAODVM2ULb7oxJijpnbCXeF1dndkjxzh+bzj+hLjrWLP3RPjXb0iq7VaZR0yF1tgIW2YmlJoahEq2oem3/2H9UUfD3isPgU2bAAA1b76J0nvuTXq5SnW1GCju8y8AAE2//4H0Aw9E9l//CsnpxJoJE9Cw7DuxbS0MJtoTtUtQ9vHHH8fgwYPhdrsxceJE/PBD8/U3Fi5ciFGjRsHtdmPs2LH48MMPI57XNA033XQT+vTpA4/HgylTpmDt2rUR81RWVmLmzJnIzMxEdnY25syZg/pWnuwTERER9QT+1asR3LYNwW3bYp6zZscaFxN1S5bEXY4WDMHRty+8fxE14Mruuz92nlAIxZdfgcCmTXD072+WHmhtHdTyhx42/5YsXfuT7RqnNjZC9nrF4FTNZMu0d/0z9+jRLc6TM3Nm/ItJux3+dWKglh3KRtE0WIvkbvy/GWKy9SKquYHA9PdRstvNi+c+CxYg6zh9RGtFgS161GqbDLW+HhuOP7717SUiohjuESMw6o/fzWArII55ESQp5iabFkp83NAUBY4BA/T5mg+secaPb12DE60zGO7JEh0wlmQbKh57HMWXibqvSnV10st19u+HUSuWw5aRkdT87l13Nf826rsHN0f2CCqac7Zop68JwS1bAABp+0yMmMc1fDi8e+0JWM4rjEHHtEDAPG67d9utxTb5o+Jf8QS2FKP+88/Nx/b8fLEN+nYrtXVQaqoBAI5+/QEAtYsWiR4tSQ6+uf3pp7HtxpsipskeT8y5WLC42OydRJHaPCj7+uuvY/78+bj55pvx888/Y9y4cZg6dSrKysrizv/tt9/ilFNOwZw5c/DLL7/ghBNOwAknnIA//vjDnOeee+7BI488gieffBLff/890tLSMHXqVPgsg0/MnDkTy5cvx+LFi/H+++/jyy+/xLx589p6c4mIiIi6lOo33wQABDZsxLpDDo15Xgsp0BQFgY0bzQDtlgsuhFIf54RdiRw0K6RnllgFiopQ+8EH8K9eA+eA/pD1zNbWXFABQO0HH5h/yx5RU7X39dfD0b9/Uq9Xqqthy8oSI1DbEwde49WNa0uuIUNanKfwxhvgHDgwZrpks5vdUXc8KBuHau0imvhiXLKMrmxc7Kfvvx+cgweb8xgZSgbX0KGoee+91reViIgSkux2s+eDa/gusUE2RYm96agHPlfuGicoqCjhm4HNBNa8EyeaQb+dpVkyZf1//onGH38MP2mTEarcjsCGDdAUBeUPP9KqZSeqEx+Po08hACDn9NORppciAIDaTz4BAFT8/e9o+OYbAIDq8yO4bRsKrrwS3okTMfSjDyMCyplHHom8OWfFrEMLBqEFAvCMGxdxfNc0DcGSEqyZtG+47W43Kp54Ak3Ll6Pp998TtjtYXBx3ui1TDDa26dRT4V+9BgBQt3gxAKDpl18heb2xQfxE69i6DfWffSYeKCE0/f5H5Az6eUWoqgpyenpETxkS2jwo+8ADD2Du3LmYPXs2xowZgyeffBJerxfPPvts3PkffvhhHHHEEbjyyisxevRo3H777dhzzz3x2GOPARBfyoceegg33HADjj/+eOy+++548cUXsXXrVrz99tsAgJUrV2LRokV4+umnMXHiROy///549NFH8dprr2Hr1q1tvclEbYKp/olpiS4iqUNowSA/E6IU0wKBhCfXO2LDSdPM/bTq9TcAAP71fwIQmRVrDjggPLOqILBpE/484kgxMrN+kbdmwoTYdoZCkIw6a6pqduGzMi4Mt155JeS0NLMum9bki5k3WcYgWK5hQ5uteWqlVFaKoGwoGG5zJyA5nRj1+/927LV2GyRIyDzmmB0LysoyoMVmwvr/XI+gcQ7dXKasbMmUTXDe4h4TmQlsy8vD0HfeNrOriYio9eKdexs3x2RvWtzMx+igrHnciPP7ramqWde8uetSyW5P2cCNWiBotjFYVBSxDZJsg2R3QA34zRqubUVOE9mxBVdeETE9WCyOi9aAcGD9n9B8frhGjYQky+JGa/RnE3UczTr+OPO9VxsbImqsaz4flMpKKHpvIk1VIXs88K9ejaZff40IyirV1QiWlkHTNNQtXYrgli3oe999sduTnhZub1ERgHC5KU1RUPHIo9B8yZ2TWb9DZffdj40nnwy/tRaufs6oNjbCs/vucCXRG6inadOgbCAQwE8//YQplhHwZFnGlClTsGzZsrivWbZsWcT8ADB16lRz/g0bNqCkpCRinqysLEycONGcZ9myZcjOzsYEy8XClClTIMsyvv/++5RtX3e09pBDzVR4pb4eqs+H2kUfJ/zh9f/5J+qWLIkcsCGKpmlQ6uuhNFM+ouadd1B6z71xs26MdgCxg1b4162LOACpDQ0Rj5X6epQ9/DB8K1YkXDcgRm72rVrV7Dy+FSvQkOB7m+jAk4qRJzVNg6Zp2DRrFiqefDI8zVrXR1VRY8kYUgOBuNlJla++isZffjEf+1avQeOPP0YuS1HM9zta0/LlaPguch9q+O47rN57olmDJqLt+uiT0V1Sg2VlEV1kzcFAQqG4mVJKffhz1TQNxVdeBZ9+Vy+4dStWj98j4SigDd99j+o334rbLbbxxx+x6fQz4gYNmv5Yjop/PIXtTz8NTdOgBgLYesMNCJaUIFRZCf/69XHXZ25T1D6jKQpq3nsPocpKqD5fxB1TTVHMx6HycqiNjVDqG6DU1CBYUoLaRR+b8/r//DPms9U0DUFL7wPV70dA7zaTLKW2FrUffZT0XVGDUYA+WFKC9ccdj8DGjSiefznK74/tsmxo+mM5qt/6DwDAv34Dis6e2+JNBzUQML8DSk0NAhs3Jt1GtaEh5vOP95kbGn/+xVx+3ZIlcQvda4qChu++a7aek1JXh2BZWdxtC27bhoDe7cm3cmXEaKd1n34KRT+51AIBhKqqEga5o5dt7Lt1n36KYGn8HimJaJb3WAsG4/6um6PVa5o5Ci0gTuhi7oxbl61/v5X6hrjfMWvwKFBUhA3HHAvV8p022qUGAhHr9a1eg+LLr4h5f6rffAtN/xNBLdXvN48taiCAmnffhVJbC6W+AQ16OaX6r7/BthtvMpcTqqxEsLQ08fZomhkgVeob4F+/Hpqqmt3GI+ZVVQSiLiSsx4ZE61Gqq82MENXnQ9Hcedh05uyIeZp+/x1KTY2Yp6HB/B1UGxrQ+PPPYl2KgtJ77zXfY2MbfStWILBuHeo+/RSyWy8fUFYOAPhzyhRRf874PoQUc0Tm0jvvEnVYE703oRBgdDVUFKi1tVBqayO33/LdktPSIEkyIMsRn21rmaMhxxlEIxGlod7M2uhsowHvcMkEux2qzwc5PW2HgrKSJIUHQLHY/tRTKH/scQDNZ8rCGCTMYY/Iro1YhyPy+2PLzIKjTx/xWRAR0Y7RNHFjzUo//5LTvOZx3OAeMwaOwj6RiwiGwtdlMaUNQpCNoGyC8gWaorTqONwctalJlEqy2zF6VZya9zYZks0GLRA0z5vbiqTXrZejzn/K7r475hy0/OFHoAX85nsFxLkujDqO5p55ZvjcSLZFHD9Vnw+qLzJIa5YfqKqGWhve9rL778eWCy6A5vdjy3nno/Kll+AZG5v1LHu95t/WMgy9b7oR9rxcuMaMhmePPeK8E3HoN7WN0hay14thiz8xn65fulRfURD9H3kYuaeemtxye5A2PQOtqKiAoijo3bt3xPTevXtjVYIAWElJSdz5S/RgnPF/S/MUFBREPG+325Gbm2vOE83v98NvuSNRq49op6oq1BQE1jqz+u+/h++339Dwl7/Anp+P0vvvh6NvX9R/thShigq4R41E8aWXIv+KK+DedQyKL74EA557FlvOOVcEQX0+eCZMQOEdt8O3fAXSDzxAZMqFQqh65RVU/+s1qPr72f/pf2LbVVcj85ijUXDNNdCCQWw5/3w0fvMtvPvtiy0XXIABzz2LwMZNgKbClpODdfvuBwAY8uEH2HDU0Rj21Zdo+Ppr2DIysPWKK6H5fPBOmoS+Dz6A9YdPhVpbi7wLLkCvC85H7ccfQ6mpxbabb0H6IYeIA5VNRqi0DNUvvwzn0KFwDhmChi+/hGv0aGSfegoavvwSuWedherXXoPv99/h3XtvSE4nat55F5Ldjsxjjoa9sBC177wD7957I3vmTGw44ki4x46F6vPBPWY03GN2Rd0nnyBUUgLvvvsiVF6GjKlTIdnt8K9ahfrPv4B3773R+MMPyDrpRAS3FKP2vfeQfeopSJs0CWX33ofMY4+Bd8IEbL3yKrhHj4JjwEA0/vwLaj76CFsvmw/P3n9B7qxZ8K9fj4bPv0DTTz/BNXo0lNpabJ59Fjy7747smTMhORwou/NOuHcfi+DmLaj7+GP0uftuKHV12DRjBmxZWUg/+CBkTJ2KYGkZyhYsgHPoUAS3bIHm88E9dix6XXoJQlu3ofr116FpKuwDB0B2uyF7vaj612vIv+JylD/2OPrc/TdUPPoYPHvuAc3vR9ldf4MtJxuQZNgLCtDr4ovhGr4Ltt14EwLr18OWkwP3brui/vMvkHvmmZAz0lFy7XXodfHFgE2Gb/kKBNauRWDDBuRdcAGyp58M3/Ll0DQN2264HoV33omyO+9C7xtvxLYbb0LWSSdC8nhRcsMN6HvfvfAtX4GGr76Cd599sHneOej74AOoeuklOPr3R9qBB6L8sceQc9ZslNx2OzRVgXPAAKQfeigaf/4ZDV98iZwzzoB/7RpUvfY6JJcTyvZKbLnoImiqBjnNi/xLL8X2fzwF9267wZ6Xh8aff4ay555Yc++9kN1u9Hvy76h68SVojY1QA344Bw7E1iuvgqN/fwSLi5E7dy4gAQ1ffwPvhAnwjB+HqpdehpyRAaWqCra8XMhOF/wbNiBUWQk5LQ3brr4aaQccADk9HWn77wf/uj8RKilB3Ucfof+zz0Cy2VD5z6fR8PXXSD/8cDgKe8Oz555wjRyJ2vc/ACSgfslncI0ahfSDDkTjd9/BM2ECtl1xJXpdfDGqXn8DfR95GBuOOhq2zEyESksx8NVX4Bw0CJXPPoum//0PeXPnAnY7qv/1L9S+/Q4KF9wBzR9A+uTJ2P7ii1CbmtC0ahUaV65E5TPPwp6Xh/z5l0FtbMSWuXOh1NbBlpsLW14u6r/4As6hQ7BxxilwDh0CzR+Apqrw7v0X1H20CK7huyDv/POx8bjj4d59d0guJ9S6eijV1XD07y+C+JqGwMaNcA0fDi0UQu8bb0TNvxei6qWXkXf++Wj66SeEKiqQdsAByDrpRJTccCMCGzdCra3FgJdeFL91JSXIOW0mmn79DZXPPw9bejo8e++N4KZNCG7dipyZM+FfswZyehpy583DlrPnwjlwIIIlJcibezaqX3sdvS67FIENGxDYtAnVr70Opbwc7rG7QamrR9a0k1D3wYfIPWs25PR0VL38CtSmRuRffjm2P/oYlLo6pE89HFXPP4/qhf8GFAXZp54Kta4OjT/+CKW2FjmnngJH336QPG64x+yKLXPnQk5Ph3OXYdCCQWQecyzK7rwTuWfPQe37H0BO8yJj8mQ4Bg6CPS8X5Q89DNfIkWj4+mt4xo+Dc+AgePedhFB5BWrfeQf1n3+O3DPPRKCoCI3ffw/XLrsg+5QZqHj4EfR95GGU3nobAn/+if7PPI2Khx9GYMNG9Lr4IkgOJyqffRaB9euRdeIJcI0aBUdhIapfex2558xD/ZLPUPfRR5CzMqFsrwRkGZIkIf3QQ5B54olQysuxefZZyJ03D1pTI7RgEGmHHoqG519AaX4+tKYm1P7nP0g//HD9d24U0g8+GBWPPQ5HYSGcQ4Zgw19Phi0nGzmnnQ7/mtWoeOBBuEaORMEN16P6tdfF5ztrFsrvuw/2wt4IbNuGYNFm+NesRt7556Pq+Rfg2WM8tl59Ddy7j0X1GwsRKinBoDdeh2S3Y+NJ0yCnpYn6o4GACDLW1GDgKy9j+1P/hH/1amRMnYrGH36AZ/w4ZE2bhuLzL0DG1MNhL+yDukWLIDmdGPDiC2hctgxbzp6LwrvuhH/FClS99DKyZ86Eb8VyaI1NyJh6OLyTJmH7k/+Asr0CfR99FL7f/4B77G6QszIRKC2FpiiofOZZ+FeuBOw29HvsMRRfeCH8q9fAUVgIW24uAhs3ot+jj6L+08XwrViJ2sWfQvJ6UXzuuRj+038BAOuPPQ4A4N5tV2QcdSQCf0bebDIyNQJbi6FZLvQkpxOa/pwSDEZ071eDQUC2QVVV8yKkeP7lsBf2RuFtt4nXWDNi3R5oEPXHlMbGhOdcG044EYP/81ZEfTwrTRPnbJrNDtXvT+7cTd8mNRiCJsvNvmbASy92jfNBWYba2AjJ44XqD4j3RNOSb7vdDi0Uiju/GaTXM3ziLlMf3dsYACXePFKaF4PeehObTpqGguuvg3f//cR8koTK115DqLxCZGdv3AQ5PV1kMWsaNFWDe9cxkOx2BLcUQ87KhOx0IVS5PTw4SjAEe2FvKNsroTY2ikC/qsCWlwdbdjaUmhrY8/IQKiuHFvDDlpUtbvirCiDbILmckF0uKHX1sOVkQ3a6oDY2wDFoEJTqaqj19bDn9YLkcEBtqEeorAyQZGjBIGx5udB8ftHWYAC27BzYC/LFjdgmH1RfkxjMTAKgKJDTM6DW14kLcUlCsKQU9rw8yF4vglu2wDViBILFWwDZpo9A7hY3PEIhyBmZUGtrIHm8ogxHRjpUnx9KZaVoS042HH36wL9mrQjQB4JwDh0KtaEBakM9glu3wtFXlJHQAmJfltMzENy6Fc6BA8XNYr8Pclo6IEsIlZbBtcsuCJWXiWCMpkLOyIAkSZCcTig1tSLDWtOgKSokhx2higpIThcchb3FcXrzZiCkwF5YCM3vg9rYBNeokZDdHgSLt0ALhWDLzgEkCcr2CkgeL2S3C1owhFBZGex9CgFVg5yWBt+KFbDn58MxoD98v/8BR2FvSE4n5LQ0BEtLIdnssPfKE4kBW7dCcnsASYLa0CBKbDgcsBcUAKoKzS9+0+25uVBqqqH5/ZAzMhEqL4Ojf39o/gDUxkbY8/OhBgJorKpCmdMJtbISziFDAElCqKwUtuxs2AsLESopgerzi99ITYW9b1+ESkrF+5meoW/fdkgOOyS3R9xoVxXIbg8ktwuSwwnJ4YDkDP+vNjUisHET7Hl5UGqqIbncsGVnQamshORywzmgP5TaOoTKSkVdSE1DqKwUktMFzS9+b225eaK0iapBdrsheT0IrN8AW2aGCDipGiSnA0pllciG1DRAUyG53JA9bvjX/QlbZgZsuXmQHA7YsrPQsOw7uEeNEt9Prxeh8nKxP4VCsOVki8/TJkOproFjQH8oFduhNjTAOXgwlKoqESgMBMxBHrXGRnF8CSkIVZRDTkuH2tgIR6HoOq7U1kKtq4Pa1AjXiJGwZWZCbWiAvbC3GZgKFm+B2tgE55Ah0EIhhCoqYM8R+74tLw9qXR2cQ4bAt3KVqHupH88khwPQVKg+v6i7qWninLOhUayjdwFCJSVQqqrhGDQQan0DbBkZCFVWQmtqgpyeBnvv3lDr6sS+3dAALRCAnJEOSbZBU0WwMrh5C2w5OZBssti+gB/27Bz4//wTclqauEEe8EPOzITW5IPkdkHT22T0VpCcTvG9qKuF5HQBmgqltlZcP1RXR/zuqvoAipKeKWt9btC/F8b8TqvBAFbppQtCdXURNVfVUAhwOfX5gnF/3xt//BH1S5ci7ZCDmz3uJHNsalq9Go3ffw9ccnFkG42/JRmQgNC2bWhatRrOXXZBYN26tjlW6wFW67I9e+6Jpp9/RsASXxr2xecoufEmKD4f4HSG57c7IHk85mMjqF244A7xXtjt4ZvnwSA0RCaZKXpCg6qqCDU2mjcxK/RrWXM9LjdUn89sk3/VKtj69499T5xOFN5+O0puvDEisceWXyCORXosLZn3UvUHkHHkkebAoJLLBc1Sr7jXpZei4qGHxLbZbClJWusKWvM97FxpAR3orrvuwq233hozvby8PKJWbXek9OmDpqWfo+SBB+G97lr4//UvhEIKvA8+AFsfcffMtX07qu65F9r77yPtzgUomnUm0u64A5LTCfuuY9D08svYdOZsOI88AmVPPQVl82bYR4yAlOaF55KLoWzZAvuIEdhy9lxkPvcs6h5/AlW7j4Nt1Eg4JvwFWZdcCrkgH/U334INV16FwAcfwDZsGJQ//0TaLTcDTic2HHU0XKfMwJ8HHGi2PeORR2AbNRKBTz/FuslT4P7rX+GefjKqzjkXVf/+N7T6emS+/BLU2hrU/P4/aPUNgE2Gc/JkpD/4IBAIQEpPQ9YVl0PZvBk1v/4KNTMTm/5vBtL/dhdcxx8P/y+/AA4H0p94HFJGBvzLlqFx2zY4r7sOTd99h5oLL0L6A/fDNnQotPp6+D/4EE3ffAPn9JPhzs+H79MlkAYPwbbLr4DrhBOgbt+OtEceRnDlSrgOOhD1q9dAGr4LMt78NxpefgXb55yNjPvvR+1P/0X5k/9AxqOPwPfCC3DPmgXtl19Q8sijyP7oQ4RWrULZwoWQvGlwnn4a0meeik3zzoGUnY2MJ/8OuW9fbJv+f9CqqpC96CM0PfMMHLNnI/S/3/DnwYeI9++f/4Rt2FA03nc/6u5/AI7x4+CZfxmchxwC+6+/Qi0vh5yTi633PwA5Oxv2gw+CfcgQbDjueMDlguRywTZyJOwHHgjfd99hw8WXQMrMQOW118E2cCDSn30GcDohSRKCP/+MTWecAduwobANHwHPuecAgSB8ny+F94H7UfngQ1DWrEHW22+j/u23IffKg+OceXBVVsLtcKDh66+x/aCDYRs5Eul33QnbypXYcvMt8F5xOfx9+0Ktq8W2e+6FVl2N9DvuQPEdd8C+227w3HoLQrIMe24Oiq66GvZRo+D3+VA+/f/gOWceGkePhrJpEySPB/Xffou6f/wDkscD7+23ocnjgTZyJGpvuQVSZiY8550H5csv4Jp6BLSqSpT86zW4Tj0FvrJyaJXbIZ94Ahovmw/vNdfAPmAAtlw2H84jpsIxYQJssg3SqJFInzgRalERvAcehMoTToB7zhy4FyxA4ysvo+qy+cj++GMEPvkYjuEj4HvhBUgeDzwL7kDV088g8NlnSL/vXiAYROB/v6Pp9TdgGzMarjNOR9bZc1D68MOQCwvhOOZoZN9wPXz//jdCIQWNS5dCfftt2MeNg7p5M9wL7oDvtdfR+Oq/YOvfD7UPP4KshW9Ayc+HlJWF9SdNg3vOWbANGADp66+x5Y4F0LZvh9ynEHL/ASh58SWEfvsN7jNnIfvTi1Bx1ZVQyyuQ+fcnUHP6GXDsvTekjAxsuepqeOachYaln6N6/wOg1dUh/a474RoxApAkbLvmWkjp6ch44H44NxVBWbdO/HaMHIHtTz8Nx0EHo2HDBlRPOQwZjz8GraoKSlERbCNGwDliBAJffQUppMB5+GFwhUJQ1v0JKTcHRZdcAvdfpyHz2WfQtG4dbGfPgXPoUPg/+ABbbrsdnnlz4R45ElptLYovuRTumTNhP/AAlC5cCCmvFzLeeB0IheB/9124zpoNR2Mjqh55BO5TZyLw668ouvwKuM49R7yfX32F4gsvgvPYY1B8512w7zoGcl4vpD3yCKT0dMgZ6VCrq9H49ddwXXUVyp94Arb+/eG5+WYomzZi24MPwXPmLNhsNmw8eTo858xD9icfQ6uqQtObb8I+aRLS518GZd2faPzic0hVlVBXlMB/3fXI+PsTUEtKoDb5YBsyBBUvvQj3JRej8tV/IeOB+6Fs3YqGbdug/Pgj/O++C/dpM+ELBuG5524oq1ahoaICFXPnQe7TB7Zhw5D19n9Qe/sdcB17LLzHHwfN50PZM8/Ac+ml2Kz/fqVdcjE2nXgSvNdeA+/lV2D7E49DLSlF2k03wr55M/ylpaj959OQ0tLgufgibL3/ftiGj4D3maehlpQi9OsvgKbBsf/+qL35FtStWgWtvgHp99wNX2Ul1OIm+N58C5kffYjACy/A1rcP1FWrkfnyS1CKi5Fx5RWou+hihFashOf66wCfD1peHoK33Q77KTNQfO65cE2fjqx33wF8PhRffAlcJ5wAx2kzsfWUU5Hxz6dgGzYMVRdeBNvQIfDedx+2nT0XzgMPgDpjBkKffIL6YAju668HbDI2X3U14HQi7a47AUmCbcRIBL/9BrYRIyA5nCiaNg3u006Dd9Ys1L/wAtIeehCNjz2Gxqefgffhh+H//X9oXL8e7jsXIPDBh1g3eQrU6mpkvfcuyq+5FvbRo5H90Yfwf/AB3LffDi0QQO0//oGq9z+A5/zzYWuox4Zjj4Pcpw/S770XUqnISJcH9Idn7lzx+/PWf7D++BPgvXw+MvfaC2pJqfiNGTgQm6+4HPbhI+C57jpsO/10aNXVsO+5B4pffx223XaF8sdyAIBv+Qo4Lr4YgauviTg/2PqFGEW37I4FsI/b3ZyuWbJKyzZvgeT1mI+D5eVQfD4oZWUI6OdP/vp6BMpt5ngCgbIy2EYMh7JmLWoCfgQbGqA5nfBVV6OsrAxqbS0kWYZkyZwMrFmDsk1FEeuyqinZBn9ZGUL19QhWV0OJM3aB1tQkjkl6EDmgZ6VI9XWoqK5uvvvYgAGoTzAeQmcSaGxEqL4eTdDg316BhrIy1NTUQNM0yEnUz9MA1NfWimBjFJ9PZDI31Ikb7fHGh2jS5/HrAfnoebIXfYQGd3hQtqb+/RGsqwPq6iDPPRv169dDHrsbIMuwHXMMNJ9PZFnr37n69euBYAjyuHHQamughRTIu40VLVdUABp8VVWQR44QgSanC7DboJSXQ2tsgtS7N3zVNZB33x2w2xBoaIA0ahSk9DQgEBCZ3qoKKS1NBMv8fkgZmfBtLYaUkwOpf3/4auuAUBBSbi7k4cP1zDAXQjU1ImNI0yB5vQhsr4RWUwO4XZDy8iC59awpSQQFtYYGSOnDRTBI1SDn5UKprkGwqhK2gw9C0+bNkPfaUwTL3C4oPj+ghABZRqi2FlJGBjR/AHJONoLl5ZAyMmHLzhKxtNoa+MrLYT/iSGh1tZCcTjStXAU5LxeSxwNH335QS0SPJcnhAGw2hKqqxAV5eTkADXJWFkL19VDr6mA78kg0rVwJebddxc0MhzPcAy4YgJSRIQIksgTJZoPa0AA5Nw8IBBCoFBn88mGHQ/K4oZSVQfJ6IckymjZuEgGofSZBsskINTWJ93/CXiKo6fcDsg1ybg4C5RVitPXqathmn4lQaSkClZWwn34aApVVgK8JWmMT5JEjoQUC8Fdsh2S3ic/a74emfy4IBAFFQbCiHLDbReAzMwvByu3m9oXqGyDlHQDfthJIHg8kjwfB7RXQHA74GxrgyMuDLTMTTVu3ARIgjx+H4Pbt0CorIe2yC+T0dBGcdjlFkHjwEEgeD5SmRsBmh5yTLQJcjU2QXE5ImZlAKCRuhAWDQDAkBsMJBkWgRpZhmzwZoYoKyPkFQMCPYE0tJK8HmgY0lpRAGjQYct5B8G8rEZ/DpH3EssQdKyh+P4w4j+b3AY2NkP+/vTsPk6I89z7+q+pt9g0GhmEHWV0RI45HY1QUlOR1IQaVI5oYzTGQaHBJPMeISzyaRGM0QXNiEvUkIieaxC2KISiaKEIkagwqQQQRcBi22Wd6q3r/qO6a7pmejenZv5/r4qK7tn6qZ56pqrvuup+zzlK0rlYKhpzvNhSSMXacM2CfaTgDQdXWSqGQPBdcoEhtjULV1bLr6qTqannOPVd1u3bKyMySXV8nY9pUeebMkeQEksL79zl/y7OzFSovl3nUUTKyMlX78Q4Zh02UXVvn7H92juT1OH02FHJ+5nm5Tl/3eBQ+sF+GxyMjJ0eerCx5MjMV3vKhQg0NMrIy1VhRIWPUKOdncdxxMvx+Ne7YEfvZZTj9PCdH0dpaGbm5qt2yRZ65cxS1nYxFIyPD+a4MQzJMBaurJNN0grWZmZLXq/DBgzKPOEJGfr5CH38sIy/P6YeFRU7iRkODwhV7ZUya7PY5IytLdm2dbNuSYTrfr3nkUYrWVDs3cKIRyeOVfeCAPJ89WZH6ehlZ2ZLfp0hlpRTIkMIhKRBQaN9+mdOmOm2M/V4YmZmygyGnbE12jiLVVQp88YKkv7vBSudJsejkSbIOVrY6ro+7fF29+6j93p27ZA5pGpgxsm+fQrEnKSpjx5cW68fOK0KNwTY/y7Ksdo9NkVj268GaGtUkbCu+3cZQSMHdzt+xfc88I/Ozn5Vx8GC7+3go4jcO4tv2TJwoc/750t//roq3nWCkOWqUDti2QuGwogcOKlxbJ09seXvqFOX/7nfu+vVVVTLy8tR4wgkKVlQkPSkUqauTHQzJV1Ymz5Qp2rtli6zYU5IVFRWy9uxRNKMpC7euosLdbkM0otCHH2rb3LPc+Xv37m2xP/tqaqQTy5TxlS+r8VcPu9NrfF6FDhyQFXRuFm4540zlLv+pzIKCVr+b+vJy+c88Q75Jhyly/09kjBmd9DOIzpolc/RoWZ980i0/m76qphPZ24bdjYX/QqGQsrKy9OSTT+rcc891p1966aWqrKzU008/3WKdMWPGaOnSpbrmmmvcacuWLdNTTz2ld955Rx999JEmTpyot956S8ckjOp3yimn6JhjjtF9992nX/3qV7r22mt1MOFx1UgkooyMDD3xxBM677zzWnxuqkzZ0aNH6+DBg8rLy+vaF9HHWZalvXv3qri4uM0T9sjevbIjEflGjHBObBKyVWzbll1fLzM7W5GKCpm5uTIzW144WcGmVP7gRx/JN2pU0mMAda+9prq//EXDvvMdRWtrVfn4405WXuI26utlBAKqe/VVJ/s1pnHTJgWmT5dhGKp+4QVlHn20fKWlh/Sd2JbVqeLf7W7PthXeuVP+WFp/Z9ZrLSsolWh1tZNVEmt7ZO9emTk5ST+LaG2trOpqeYcN6/LjmqGdO2VmZMg7dKjC5eVq/Oc/ldus/EjS8tu2Kbxzp7JOOKFlDSPbVvTgQXmbj8rcSYm/Y+lSfuttMnw+Df/PG9v+7GZ9yWpokJGR0erPMLR9e9LgJ5EDB5L2P34h6j4q1Mnfh0Nld+BRXjsadQMbVp1TasFXWtrquqHt2+UtKXEfk5akXYuXaOi3rlHgsMPSuwOdYIdCbT6KnfbPS9Pflp76XTiUzw5/+qk8eXlu/a1Obz8alW0YHTouNWeFQs4Fdqx9digkxd5bDQ3u38LIwYPy5OfLME0nCz0QSNneyIEDsqqrk/pp0vz9++UdMqTj7auvd9rXiUfT45kT8XVq/vQnZRx+hHwjneObHY3Kqq9POYJw4wcfyFdaKk9enpPlGDsW7bz8q8qcOVPVzzyjkQ8+oF1Xfd19GqU1vtGjlfO5U3Tw179xs8EkaeJf/5L0d6vujTcU2rJFhZdcoh2LLlXDm28qc+ZMmTk5GvXgA84+vPiirMZGld/4nxr37DOqeWGVqn73OwWmT9Oo5cu1498vUeZnjlPx1Ve72908/XBNWP0nVfzwh8o6/ngVXnyxNk9vGg15zMrHlXnUUQp++KGqn35axdde22Iftp4+W8XXXKO8L3xekrRz8WIZHq+MgF/Dv/tdeQbA+V7Nn/+sT79zo4Z+Y4k8ObnKPe/cTvWlD0/+rAoXLlThoksUPXhQH51xpjsv79xzVf3UUyr62pU68D8/15T3NrVYf9/992v/z/5Heeedp+o//CHlMnGbpx+usb/9vw6NMg30BR29ZgL6ioMrVmjv93+gye+83e6ym6cfLv/EiQptderLT1j9p6SBGRv+8Q9VP/dH1b36qoZ9+4ak6/C4mjUvyaqsVM2aP2vUAw+0+lkd6UuNH3ygj8+fr7F/+L0ypkzR5umHa9T//EzZsZr3lU8+qT03L3OXzzvvPI2443vt7uehsOrrtfeeezT8u99tat/mzfr4vPNV8t93qPw//0u+kSM1YfWftHPxYvlHj1bBRRfJP3Zsyu2V3/RdBT/6SGNXPOZOi5/TeIYMkX/8eHny8pR9ymflKRoiq6pK5TfdpCnvbVJo2zbtvffHqv3znyVJuXPmqPTeH0mS9tx+uyofX5n0Wc2Pw1VPPaW8c86RYRja/9BD2nfvj915E15ao49OO10ZM45RePvHih48qPEvPN/qfkjSzq9/XSOXL5dhGIocOCBPXl7Ka8HevHbpDdXV1SosLFRVVVW78cRuzZT1+/2aOXOm1qxZ4wZlLcvSmjVrtGTJkpTrlJWVac2aNUlB2dWrV6usrEySNH78eJWUlGjNmjVuULa6ulrr16/XVVdd5W6jsrJSGzdu1MyZMyVJL730kizL0qxZs1J+biAQUCBFIMc0zUFx0DUMo9199TcrGdFC7ILQH3vEJJXE4GBmikBM7sknKzf2h9bMy1Px177WchuxzJm8009Pmp515JHu64J5rV9Ydkg3/Mw9bfwxS5fmd7FS/czMvDwpTReeGQkjQwZKSxVoJwieMXGiMiZObHW+Jz6iZxekuhnQVaW33tLhZRP7UntBqYwJE5Le+5vtf7pGLu20jgQpE/qImZsrb7ytrazbfF8lafSDrZ+s9ZiEIHGPGATHk0CzkdU7LfYYeUeOSy1Wbf7zTHif2B/9CYHU5v0ukX/oUKmt+cXFHW6bpEOrmdns3CR/7txmGzXlyc9PuWrW9Okt2zBmjMK7dsk7okRTN/3Trb3rSagvlkqkokLeYudxtqQbGY2NKv/P/1J22QnKP+ccGVFLhtcr0zSVfWKZGt58U3Zjo+zYNElSOCxP7GfjjT0CLUmG7Zx3RQ8elIKhFj97u75eoX9tkX/U6KR5+ef8P2XHzgk9GRnO490pfm8in37qPDYem2fIkOn3OdnmPt+AON8zvT7ZDQ3KOOww1a9fLzNWLqSjfcm5aWSr6skntffH9yXPi/+cYplSqbYXv+lkxm4itPeZhmUNiO8dg8ehHJuAXhOJSJ04vsUDspJkVVXLTEgoMmxbvqFDNfRrX5Na+dttRCMyA34ZRvt9pL2+ZMRK5XgCAXcZ/5gx7mtPs2u+7M98ptv6pZmToxHLliVPiw1sGdrijCdgZGTINE3VvbxWdZKGfOUrrbbHDoWcUoDN5uedfbZq1651zmdNQ56sLBnhkKzaGhlZWc7yoVDSjXi7sdHdjlVVpaGLF2vf8uUKTJqk4JYtLT6j8Pzzm15/6Uvad++PNfHFVap6+mn3es6uq3cHFjNaOaeKMwxTnliiTlvn1INNp65furEdkqSlS5fqoYce0qOPPqr3339fV111lerq6vTlLzuDVSxatEg33tiUfXb11Vdr1apVuueee/TBBx/olltu0ZtvvukGcQ3D0DXXXKPvfe97euaZZ/Tuu+9q0aJFKi0tdQO/06ZN09y5c3XFFVdow4YNeu2117RkyRJdeOGFKj3EzEkAAID+zjBNHfbKWo28+24ZHo+bHdreDS07GHSXNfxNmb5bZ5+hqqeeUuUTTzqDO0bCMrzO/OKvf11Dvnq5rPp61a9f79Yts4JBtyaemZXVclRi23ZH602aHKu9KTUNmpE5c6ayTzqpaf98PtWseUmN//pX6v33JeQjGIYMn192Y4N0qANr9TGGzyvZtjKmT1dwS8uB59plmk4ZgKjV8ucS097AjE472v8+888/33mcHADQLQITD1P+vLMPad3tX/yiPvn64qYJluWUSvB53QHEmrPD4Vh93q4/jB1/UijxhruZ1XSDPV6XNavsBElSfuwpmJ7iHTpEGUceqZpYxmrzxACjjSc3iy5dpOJvfqPF9PzzztPoXzzklMExTBk+n+xwWA3vvKOM6dMkOQN/xc/Zso4/3ik3EhOtrpG32AmMjv/dk+3vQ2Gh0/a8PBV/85vuAGDBhHOoyP4D2trGk1Toum4Pyi5YsEB33323br75Zh1zzDF6++23tWrVKnegrh07dujThFHYTzzxRK1YsUI///nPdfTRR+vJJ5/UU089pSMSHm264YYb9I1vfENXXnmlPvOZz6i2tlarVq1SRkJHeOyxxzR16lSdfvrpOvvss3XSSSfp5z9vGt0aAABgMPINH+4OnOIpKpKvtNQdWbgtngInI9fwtcyIr3/zTQU//FCKRp2BmWKMrCy3Fls0VhPNDoZk+AOa+o93UpcMiAVlbdtW4+amC4PKJ3/nbMu2ZQeDyj1rrsY99hvlf+ELTZ/n9yu8c6eqX3gh5T5YzcYJMPw+WfUNSYOV9Wfx/TACGYeWmW+asq1o24HXVuZt/fznFfwwlmXVgUcUS//7DmVMntz5NgIAOiTn5JM04vbbO7x84SWXJL2vfemlpkEeI86AiPJ43IBoc7uvv0FWfUPKeZ1lx+u+J5yfmJlN8Z7o/n0yc3M19MorneV6+Oaqd8gQjX/itwp/8onz+bFY1Njf/NppaxtB2cwjj1TWsce23OawYco69ljnGOz1OINKhkKyGxrlyXXOl+zGRvfJq6wTZkm2rZq1a502eL1ukNbw+zX1/fc6tC/x87r4OYQn4enbyN69Cn30UarVYqX60ls2cDDqkYG+lixZ0mq5grWxX6BEF1xwgS644IJWt2cYhm677TbdFhvFN5WioiKtWLGi020FAAAYLLyFhTrspTXuRZd/7FiFPv445bJupmwrFz7RykrnBD2hlpgZCCgSG2Qist8Z8McOBmUEWi+TYtu2DNOQHQxq2znnaNoH70uSKv/v/+ILxOqHtyw/Em+bmSID0/D5nMFVYp8Rn2YFgwMmKCuP892bsWzmf804VgUvrurw6obHE8uUjbSR6ZR6eujDrW6ZkMSsZgBA3zfl7bd04JFHWkyve/VV5ZxyimRFZXhMGR6vrHDrgVcr2JiWv/9upmwsuDnh2WeSSrvlzp6twNRpzk3I3uTzSeGwzNh5Tfw42FambGvcm9qRiAyvz82UTbzJajU2OjfTMzKc8yDL0r4HH9TO/7hKWWUnKP/zn1fWjBnO9jrwc8g+6SSZCYHVSetel7ewUO9PdTJzowcPtLpuZN8+eYd2rpQXWqIYDgAAwCBnGIZG/fQnmvjiKnlbqSFv5sczZVMHZa3aWu2+/gYpISjbuKlpgInIvlhQNtTKgIzNBxBt5fHIeKZsquyMeL1bM6Nl5q8RCMgOhZysk3DYubj0+WSHQ6k/px9yyzP4fDIzM50R7DvDY0pW1MmCahaUrfrDHyRJttX6Y6l2g5OJbObkKCOh1j8AoG8zMzJSDtBkxTJW7aglmW2XL5DkzEtL+YJYpmwsAzUwaVJSkNE/bpxyTvq3pOzZ3uAvLXVKMsWCw/HSAodys9fMidV0jQ2obMQCvonfZ/z8Z+SP7lHunDmyo1FF9jjloQyvc+zvzCDKY37xUNLPPV7SIC4Sqy2bilVfLzPn0Ab2RROCsgAAAFDu7NnOi1YyK+IDiuV89rMp54c+3uGs7mk6ufeVlspbOkLyeBT6yHm03QqFkjJI7EgkefDAgF92KOxmyaRiNza2mSkrs+U+GH6/7HBIu2/4tjYfdbTk9TrTQq1/Tn8T33/DMOQZ6gxo15EasO76pscJutqWm03cgmW1un68PET+Of9P4x77TYc/FwDQB6QIysaPK3Yk7ARk2yhfIEneNgb9ThRev161KZ6ajosfm412SvF4S0o05Kr/6NBndocJLzyvrOOOc28UGx0ZMDmFSa+/Jt9wZ0BVOxKR4Yudo8TOhWpfekl77/+JU1M2I0O5p50m/6iRssNh94mkdMo+6SQVLFggq64u5fzI/v3a9+DP2izTgI4hKAsAAIAmKQKaUlNQdshXL08aXCsusn+/zPx8Zc1sqpNWfO21OmzNGk149lntf+gXkppqysblzTtbI++5W4bXKzvkzLODwdYzcQxDoU92pnw0sK3MFCcAG5JV71xgGF6fopVVCu/Y0eo6/U3iQCPuI4UNDWp4++0ObsCM1Yw1Wh/IxWr9YtxqqJfkXET3dH0/AEDXxEv8JIoP3unUjPfK8PpUfsstKdfPOe005Z15Zqs3d2tffVW7brhBkhRavTpluQS3LeGQRv7onnbb7C0s1LCrr253ue5imKasUFCmv2tBWW9Rkfs6Wl0lMztHhs+n4JYtblmpyN4K2Y3B5Bvb4XBTrfdoGxnMnTTmFw8pd/bprZa0anjrLdWsWpV0PodDQ1AWAAAALkOGAlOnuiMax7l10uKP1DUTrapU4cUXyVvcVF/MMAwZhiFvLGtTch69MxNqymZMmaKsY4+VmZXp1KQNBFT3xhvavvDfneWj0aQLvLo33tAnV1zR9uASKZI840FZ/9hx8gwdKsPn05Arr2j7y+hnjKSgrDMCs7V3r3ZcvFB2Gxmu7voej/N9e9q4RGijfIGisc8YKDV6AWAQiVZVtZxoRWVHIk52rMcro63jQ1wrT1oEt2xR7UsvS5KMnBxZVdWtbyIUduq19gO+YcPkGz1aUnoGHBtx2+0quuxSGT6fqp5+RplHH+1uO/jhh0k3YBOfKmorg/lQ+EaOUvjj1Deu7djxvq0xAtAxBGUBAADgssIh5c6erdzTZydNT3yEsEVA1DAUraqS2UqGiCcvTzmnnurUig0FU2e5ZmXJqq+XEfArvGOHm8Fqh0JJg3sEP/hAklKWL2jaCUvVq17Uwd/+tmn7fmcU4+jBg/Lk5srweuUdNqz1bfRDiRdqZpZT186qdi5669ata3f9eOC6zcdF2wruxgdQa+dxUwBA3xOtqpJ//HgVLrrEnWaHQiq/9VZFKiqcgajaqhebWPs0xXJ21JLh8WjfAw9Ifr8ie/e2WkN+/8O/6jdPXJTefbeGfv0qSYeeKZsoe9bx8pWUuOc+I2671ZlhenTwN79JugFrRyLuzfDOlCvqCO+QIkUqKlLOsyPJA7Hh0HHGBAAAAJcdCstTWJBUxmDKxjeTlvENG+YO/CVJniFDZFVWtXkBZWQ4ZQmsYOqgrCc7WzVrXmpRG81qaJAnJ6fl9tq4ELAjEdWsXq3q5/6o4EcfOcvHyxfU1cnMypLkXEz4x49vdTv9TeKFmlsHsKpKRmZmmzV6Jed7thoaZIdCktl6pqtttx+UJVMWAPoh21bGtKnKPOII532srFCkYq+s2hqnzFAsQzLl0xexp1oMn7fpkfoU9v90uYK/fULRAwfU+O672n3DDdr7k582NSMaVejDra3e6O1rDNN0b0amM5DsiQVl49uMD7hpJpUvCCkwfZrzpq0B2A6BmZcnq74+5Ty35m9bN8jRIQRlAQAA4JrwzDMquOCCpFGO46UL4oZde61K77zTfe8tLFS0qqrNDBFPTo7q169X9TPPplzOzMnRnu99T6EPtyZNt2prZebltVy+jRGX7XDYqX1qmvro7HnO8r7YoF62LXk9bm3UiS883+p2+hszK0sltzoZNfHRlO2qKnmLimTVph6sI27vj+9TaOvWWFA2dT1ASW2XL4hnyhKUBYB+p+Tm72rEHXe47w2fT5H9BxTevVtWfYNzwy127GwtWBdfzw6FWs6IRloMJmY1Niq8p0LhnZ80LRZ7wqO/ZMomMlIMlnaozIQb0mMeedgddCsxC9kOh+UtdOrRpso47gqjldrAzuc6P1/KF3QdQVkAAAC4fMOHOdkpKU7GJ617XZKTdeod0jQoRcaRRypaWdlmUNbMyVUwFnBNGZTNym4xTYoFZbNbzsuccWyKpR12OOwEDxMuXOKZsjIMGaanqf7pAGJ4PCpc8CXnTSzb1aqqcjKZWxlBOS5y8ICzfCjo1uRN+fNsI/vJ/bYJygJAv2NmZsrMzGy6web1as8ddyi4ZYtTXsjrdTNk2zqmWHX1qn6+5Q1PKxRqeVyxLBmGITvhhl88oJuOUgD9meHxaPKG9ZKkwGGHudOb15T1xAYJS3f5gkTNtx1/+obyBV1HUBYAAAAteIYMaVEiwFtY6L7OPPpojX7oIUlSyX/9p6LV1TJ8bQRlc3MU2b9fUursi1SBV0mK1tTKW1Qo7/Dh7rQJzz+vwITWyw44Qdmo7ISRiN2grCR5zKR5A5NzgWsfOCAzLzd11lLi0g0Nzouo5WZCpSoR0ZHyBdSUBYD+Kx4gTfxbbtXXO09BxMsWtJGVGdq+XZ/e9F3Ztq3wp582baOmtkVJgvjN0sR65XYwKKn184K+bso/3knbtjyxJ4Xi5YmmvPO2AhMnNi0QjsiTm6PCSy5xMpG7gaegoMUgcPGg7GAPnKcDZ0wAAABoIXf2bE149pk2l4lnyhiZmbKDwbYzZbOyWh0wQmq6+Bq74jFN+ftGeWIBYKuuVv6JEzXqgeXusm0FZKVY+YJIVEoYidjw+ZouIjzeAZkpmyQWII3u3i1Pdo7saETRqqqUg684izvTG/75rvOYqlqp29uB8gVkygJA/xUPiiaWGrDq6yWvVxnTpzvLtDHoYzyAGN2/Xx+eepr2P/yIbNvWwd/8psUxyAoGJdOUEm74xW8iektK0rI/Pa07auHGM1KbZ6ba4bAMn0+Gx+PW+00338iRCu92guvxn59bp76tgd/QIQRlAQAA0IJhGO3WRjP8fvnHjevQABdmRqbCu3Yp/9xzU8+PBWUzDj9cZlaWUxvNMJwRn/1++WIXZ20N8CVJE1f/KRaUjSRndSZk5xoes1sf8+sL3EdMP9npfLfRqP416wTVvfZ60vw4M9MZ/Cy6d58a/v53Z1qqC8s2LsTJlAWA/s8OOUHZpEzZhnoZHq98I0ao6LLL2jwWxJ9EiR9nalatckvfuE9lxEUiUjSq6udfUHjPntjnh1SwYEHKQT4Hq9bOr+xoVDI9yvncKSq86KJu+ezMGTPU+O4/JEm7vrVUn3x9seygEzhPdx3bwYgzJgAAAKTWTsZjzqmf04Q/Pue+bztTNlPh3bs19Kr/SD0/FpSNb8OORCTbVvktt8r0++UpKtKY/31UU995u802ORmxodijlt6mbSWwgiGZA31wilhGq7Vrl8zsLCdzWFL04EFJ0gfTD0/KWPKVlmr8U3/Q8O/epEils4yR0XIwtTbLF8Qv0tM40AkAoGdZjU5QNjB9WtO0qmoZ3tg5gWm2mZVp+p2bp3YoVnc0O8vNfrXiWbiSim+4XnY4rPo335Qk1W/4m7uMb8SINO3N4JB9wgkquuTfu2Xb/gnjVfnk7yQ5Afbal15SuPxTGZmZSYOR4dAQlAUAAEBKRjtBWcMwkpZpKyhrZGQoUlHhDkjRnJmT7W5TSg6kGn6/DMNQ9vHHt99mn0/Rqio1bNzoZvru/clPnSzO+GN3DQ0y8/Lb3VZ/ZmZny1NQ4LzOccoXyDBk1dVqX6wWcFKd2WhEhs8nT16eeyGdMiu5jQvxeFYUmbIA0H/lnPJZlf7whxrylcslSblz5qjhnXfcY6rhMd3a46kMu+5a57gTz7jNzHIfd7cTgrKe/HxnoKrioU3blRPMpVZpS4etfbnlRNtOOTBruoy4805lHnW0GjdtSjpniO7bryl/36isY1sfdBUdwxkTAAAAUutkbVDD30b5gswsyettNavC03xAj3i9MqXO2Gy1DT6fIp+WK/O4mbIaGyVJkX17ky5aRv3sQRVf/c0Ob7M/yjr+M5r4ylpJkpmVLUUiMvPyZNXWau89P5LUdHFsNTRo/y9+6QTYTbPtUZXbyJSNl4Roq4wFAKBvy5gyRflf+LwbJM2aGQu8xZ4+kdl2/dLsE09U1mc+Izt2DK5ds6YpKJtww9WTny8rGFTmkUe525Wk4JYt8o8fl8Y9Ghh8qWrsWpYMs/uCsgXnnSvvEOdm+gdHHe1Ot61oykFb0XkEZQEAAJAWbQ1uYWZlyltY2OpJfKpRluOB04ypUzvcBsPnU+TgAQUOO8x9VN9bVJQUKPQNGzbga9UZhuHus5mdLauhUd7CQkVrat1l4kFZd1Rlr9cZLCTS+qjK8VG5Qzt3qnr16uSZsYttgrIAMAC4N2ad43ZT+QKjzRt0kiTb1rbz57tvrdrYsSehnruRkak9t92u8O7dsQnO50QPHpR32LAuN38wsKVuzZSVJN+IESpYsCBpmn/UqG79zMGEoCwAAABS8uTmauRP7u/4Cm0E44yMjFZLF0iSmZ+vMf/7aPLmRo125uXkdrgJhs+n6P4D8hYVKVpZqYIFC2RblhMoHKRZHWZ2tqy6OnmGDlGkosKd3vj++5KagrPxTFnFyxfEfp7jnniiaWOxEgWRPXsUjK3vLGwwCjMADCBueaJYANYtX2B6Oj1YZuTAAWWfdJL7PvOb33Cfronu3+98TLzubF1dy6dnkFoPHXcTb5wbPp+G33hjj3zuYEBQFgAAACkZXq/yzjijQ8uO+92TCkyY0Op8MzNL3qLC1j8rRc1Yw+eNrdvx8gXyeGTV1so3Zoysmhr5x4/TgV/+SnY4LNuKOkHHQcbMyZZVWyvvkKEKfbLDnf7JlV+T5JQvcBb0OJmy8cdMYxfIgcmTmjYWC8ra4UhyTdrY9zrxz3/urt0AAPSkWFA2/oSEW77AYzYN7NiaZjdB6zdscDJs47M9nhZPVey+/nqFd+2SVVcnIyura20fJAITJypj2rT2F+yieIB+7K//V1Pf/QdPxKTR4DsrBQAAQNplHn64zMzMVuebWZnyFLaeKZuKm5XTmZqysQvBwGFOIDHn5JMlxerYhSOD8kLCzMmRVVcn75AhCu/4JGle8KNtCm3fLin2aKphNtX8i32X8YG7hl13rTuYlx2JyEoRlDWzWv8dAAD0H/FAXMbUKc77WPkCw/S0H5SNGXLllZKkqj88pYzDD5ck+SdPlnfmTPd4PHTxYo3+xS8kSZHKSlkNDU4tdLRr3BO/VdZnPtPtnxO/ST4Yz6G6G0FZAAAAdDtvcbGGXvUfnVspFhQ0OxGUjfOPGytJ8sXrnlmW7HB4UF5QmNnZiuzfL8+QIrd8wZArvqrMo4/Wji9/WXu+/wNJscwlj5k0EIuzAeeSwVsyIiFTNiQ72BSUdWsFD9ISEQAw4MT+nmeXlTlv4+UMTLPFQF+2ZbX4+5/zuc8pb948Z93sLGUd6wwYln/+efKMGuU+El944QL3JqxVWyc7FJIZaL1GPZr02GBbZjxrumPBeHQcQVkAAAB0O8PjUeCwwzq3jt+vCc8+06lM2bj4xZ4ZCChn9umy7VhQNnbhN5iYWdmKVFTIO2SIOy37xBPlKSxUtKZGkU8/dSZ6PM6FV/OLLtPU0K9/3Rnh2W7KlI2XLwjv3u0GbhmNGQAGBjte2ibO69zUNDymZDWrKRuNuk9VSM6TLnYk4mbX2nX1MnyxQGssoGvm5SV+miTJqq1xbgwOwmN1X5Yx3SmRYMdqziN9CMoCAACgTzIzMxWYNOmQAn2J6xiGKVn2IM6UzVKkoiJpoDXv8OGS4mNqO+KZsq7YACKGYaj4m99wsqPitQUjEXeAsA9PO70pQ2oQ1uwFgAHJNGUEAu7beIBVpqdlpmw0KsXnSzICAecYETsmWPX1Mvx+mXl5Tn13SZ6cnKYNxG4GWo2Nkm1zg6+PyT3tNI351S+VeeQRvd2UAYfbDwAAAOhz8s4+W77RY7q0jZJbljkvTHNQly+IX1THM2VHLf+p/OPHJz1qmnvWXBl+v/uIopGR4Q7ykrClhPIFYdnh5PIFtkRQFgAGiMyjjtKUv21w38fLFxge031qIs6qr5eZMDhX9XPPJa1jNTTI8PudbNpYQNfw+TTtg/clSYEpU5R13HFkYvZh2See2NtNGJAIygIAAKDPGfmje7q8jcILL5QkGQG/rH2NgzcoG7soNnNyJUn+ceOcLCTbdgOzo+69N7asE1Qd/4ffK/Txx6p/442mDZmGm+FkhyOyYpmyzrx4MJbsJgAYKAx/Qm3X+N95w3QyYxNYdXVu2aCk9ZOCsj7lnjVXgcmTVN9sOe+QISr6ylcU2buX2uQYVLiVDQAAgAElnnkTlzFtuszsrBaZPIOGt9moybEL3sYPPpBVV5e8bOwC2ldSIjOz2XdlGFKsfIFTUzYsO1biwK0pa3IxDQADzaTX/ipPrAZspGKPDj62Iml+86Bs6ffvcl7En7iIRmX6/RqxbJlyPve5lJ9h+P1OrfL4cQUYBAjKAgAAYEAruuxSjX7wwVYzeQay7Ju/6w5uZvi8zuOjscynSHl5i+XjA7UYXm+LpFcjVgZCipUvCAalWLaUW/+PDCcAGHASB4ps/GCzal96KWm+VVcnT+Lx1RN7KDtx8K/ErNsUDJ9PdpjyBRhcCMoCAABgQDMMQ4bHIzsYHHTlC/ynndaUxerxaOo/3pF/1KjWV4hnNaUa+dowZcfqCO753vdkh0LOKNkSA30BwCBRMP98p+54guY3PeODgpkJy7UblPX7nExZYBDhrAkAAACDgm3bgzJo6Gaxpgq0NhcP4BpGy9GvDbnlCyTJDgWbsppij5syYjYADGx5c+e2GPSpxZMo8VrmmZnupPaDsn6Cshh0Bt9ZKQAAAAanSERmZkb7yw1QHckSNhKD1s0CrInlCyTJCoXcoKxbW5agLAAMOsGtHzXLlG1Wy1ztB2XNrKyWdc6BAY6gLAAAAAaFwJQp8o0c2dvN6DVGBzJlky6aW2TKGm75AsmpK2tVV8feJA/4BQAYPIJbtsg/foL7PtXxpr2grKegQNH4MQUYJDhrAgAAwKAw4emnlH3CCb3djF7TkaBs4mAuLYOyphRtCsoaMhTcutV5E8+gJSgLAIOC1dDgvjYCfnmHNh0/jHh9ckljfvVLZ1o7xwdPbq4TlLXtNpcDBhLOmgAAAIBBoHlQ1szPb7GMmZ+vvHnzUm/ANJzga0Kw1o5EYy+oKQsAg0XDW29p84xjmyZEIsnHGE/T6+b1Z1tjeL2qXbNGtWvXpqmVQN9HUBYAAAAYDJoFZSe9slbe4uKkaYZhaOQ9dzuvm9WgNZqXL7Bt2ZGwDJ+vqaYsAGDAs+rrk97b4eSgrOH1JM2f8NyzPdIuoL/pwBCsAAAAAPqzsY+vaJHFamZk6LBXX1H1c8+lXCfjiCM06fXXElYwJctOzoqNRGQEAu6AXwCAgc+ORt3XwW3bZAUbpcRBvTzJQdnAYYd1eNt5Z5/d9QYC/QRBWQAAAGCAy5oxI+V0wzCU/4UvtDrPW1SUOMWtHZsxfbp8o0fLamhwgrLBYLqbDADoqyIR9+VHZ50teTxJN/7aG9SrLUWXXdqlpgH9CeULAAAAALTPNGTHgrK+kaUyMzMVramREfCL4gUAMIglZM5KUmDSJJXcftshbcrMykpHi4B+gaAsAAAAgHYZRlOmrGTIk5+v6L79Mv2BhOkAgMHO8PlUeMEFh7Zys9IHwEBGUBYAAABA+0yzKfhqGPIUFiqyf7+MAEFZAEDXjXvySflHj+7tZgA9hpqyAAAAANpnmLJjg3zJNOUpKFBk3z4nKGtTwAAABpuDjz+e1u1lHnF4WrcH9HVkygIAAABon6GETFnFgrJ7ZXZhQBcAQP8V2r69t5sA9GvdFpQ9cOCAFi5cqLy8PBUUFOjyyy9XbW1tm+s0NjZq8eLFGjJkiHJycjR//nzt2bMnaZkdO3Zo3rx5ysrK0rBhw3T99dcrkjDy36effqqLL75YkydPlmmauuaaa7pj9wAAAIBBxTBN2ZYzmIthGPLk5yl6sLJLo2wDAPovKxjs7SYA/Vq3BWUXLlyoTZs2afXq1Xruuef06quv6sorr2xznW9961t69tln9cQTT+iVV17R7t27df7557vzo9Go5s2bp1AopNdff12PPvqoHnnkEd18883uMsFgUMXFxbrpppt09NFHd9fuAQAAAIOLYUhWvEyBIZke2cGg7GhUhs/Xq00DAPScIVf9h/xjx8puJCgLdEW3BGXff/99rVq1Sr/4xS80a9YsnXTSSfrJT36ilStXavfu3SnXqaqq0i9/+Uv96Ec/0mmnnaaZM2fq4Ycf1uuvv6433nhDkvSnP/1J7733nn7zm9/omGOO0VlnnaXbb79dy5cvVygUkiSNGzdO9913nxYtWqT8/Pzu2D0AAABg8DHMpAG9DNOQHQ7LDoVk5uT0YsMAAD1p2NVXyz9hguwQQVmgK7olKLtu3ToVFBTouOOOc6fNnj1bpmlq/fr1KdfZuHGjwuGwZs+e7U6bOnWqxowZo3Xr1rnbPfLIIzV8+HB3mTlz5qi6ulqbNm3qjl0BAAAAIEmmkVBT1pBMU3YkIjsSkZGZ0bttAwD0qPCuXQrvqejtZgD9mrc7NlpeXq5hw4Ylf5DXq6KiIpWXl7e6jt/vV0FBQdL04cOHu+uUl5cnBWTj8+PzuiIYDCqYUA+lurpakmRZlqyEjICByLIs2bY94PcT6G70JSA96EtAeqS7L9m2ZNtO+QI79k/RqGQYMjxe9zOBgYhjE5As+K9/Jb3vaN+gL2Gg68zvdqeCst/5znf0/e9/v81l3n///c5sss+48847deutt7aYvnfvXjU2NvZCi3qOZVmqqqqSbdsyzW4rMwwMePQlID3oS0B6pLsvRSoPyooNsBsMNupgZZUkKWxFFa80W1FB1hQGJo5NQNs6+vefvoSBrqampsPLdiooe+211+qyyy5rc5kJEyaopKSkRYeMRCI6cOCASkpKUq5XUlKiUCikysrKpGzZPXv2uOuUlJRow4YNSevt2bPHndcVN954o5YuXeq+r66u1ujRo1VcXKy8vLwubbuvsyxLhmGouLiYP4pAF9CXgPSgLwHpke6+1HjwoGptW7akQEamCocUqUaS3x9Q1O9XSGrxtBwwUHBsApIdbPa+o3//6UsY6DIyOl7SqVNB2eLiYhUXF7e7XFlZmSorK7Vx40bNnDlTkvTSSy/JsizNmjUr5TozZ86Uz+fTmjVrNH/+fEnS5s2btWPHDpWVlbnbveOOO1RRUeF2+NWrVysvL0/Tp0/vzK60EAgEFAgEWkw3TXNQ/KEwDGPQ7CvQnehLQHrQl4D0SGdfMj0ep4aBnEG+zFjJAsPjkeHxOMvQZzGAcWwCmpT+8Afaff0N7vvO9Av6Egayzvxed0sPmDZtmubOnasrrrhCGzZs0GuvvaYlS5bowgsvVGlpqSRp165dmjp1qpv5mp+fr8svv1xLly7Vyy+/rI0bN+rLX/6yysrKdMIJJ0iSzjzzTE2fPl2XXHKJ3nnnHb344ou66aabtHjx4qSA6ttvv623335btbW12rt3r95++22999573bGrAAAAwKBgmKY70JdhGM7AX5JkmlIsKAsAGBwypk7t7SYA/V63DPQlSY899piWLFmi008/XaZpav78+br//vvd+eFwWJs3b1Z9fb077d5773WXDQaDmjNnjh544AF3vsfj0XPPPaerrrpKZWVlys7O1qWXXqrbbrst6bNnzJjhvt64caNWrFihsWPHavv27d21uwAAAMDAZhjuQF+S4QRpJRkeU4a32y4rAAB9kOH3u68nv/m3XmwJ0H9129lTUVGRVqxY0er8cePGJZzUOTIyMrR8+XItX7681fXGjh2r559/vs3Pbr5dAAAAAF1kmFI0mvA+ninrcQO0AIBBIvaExLQP+udg70BfwNkTAAAAgPYZcssXOO9jlxIeUyJTFgAGFd/IkTrspTW93QygXyMoCwAAAKBdSdmwhiEjVlM288ijlHnUUb3UKgBAbzAMQ77YmEEADg23tAEAAAC0r1lQNv5+6OKvyw4GdeDhh3upYQAAAP0PmbIAAAAA2hevIRt/HStfYCQEaAEAANAxnD0BAAAAaJeRGJSV3PIFqeYBAACgbQRlAQAAALQvKVNWydmxZMoCAAB0CmdPAAAAANrXSk3ZFvMAAADQLs6eAAAAALTPaH7pkFy+YMILz/dsewAAAPoxgrIAAAAA2pc0zpeRVFNWkgLjx/dwgwAAAPovgrIAAAAA2mUklSgwKFkAAADQBZxJAQAAAGiXmZeXPKFFOQMAAAB0FGdSAAAAANpl+v1Nb1KULwAAAEDHEZQFAAAA0DkG5QsAAAC6gjMpAAAAAJ1HUBYAAOCQcSYFAAAAoHMMSbbd260AAADotwjKAgAAAOgcw5DhDyjntNN6uyUAAAD9EkFZAAAAAJ1iGIY8Odka/cDy3m4KAABAv0RQFgAAAAAAAAB6EEFZAAAAAJ1k9HYDAAAA+jWCsgAAAAA6xyAoCwAA0BUEZQEAAAAAAACgBxGUBQAAANA5ZMoCAAB0CUFZAAAAAJ1DUBYAAKBLCMoCAAAAAAAAQA8iKAsAAACgc0iUBQAA6BKCsgAAAAA6xaB8AQAAQJcQlAUAAAAAAACAHkRQFgAAAEAnkSkLAADQFQRlAQAAAHQO5QsAAAC6hKAsAAAAgM4hKAsAANAlBGUBAAAAAAAAoAcRlAUAAADQYUZGBpmyAAAAXURQFgAAAECHmVlZjPMFAADQRQRlAQAAAHSYmZXV200AAADo9wjKAgAAAOiQKRvflExTBuULAAAAuoSgLAAAAIAOMbOznRcEZQEAALqEoCwAAACAjrPt3m4BAABAv0dQFgAAAEDH2bYY6QsAAKBrCMoCAAAA6BzKFwAAAHQJQVkAAAAAHUf5AgAAgC7r1qDsgQMHtHDhQuXl5amgoECXX365amtr21ynsbFRixcv1pAhQ5STk6P58+drz549Scvs2LFD8+bNU1ZWloYNG6brr79ekUjEnf/73/9eZ5xxhoqLi5WXl6eysjK9+OKL3bKPAAAAwKBDpiwAAECXdGtQduHChdq0aZNWr16t5557Tq+++qquvPLKNtf51re+pWeffVZPPPGEXnnlFe3evVvnn3++Oz8ajWrevHkKhUJ6/fXX9eijj+qRRx7RzTff7C7z6quv6owzztDzzz+vjRs36tRTT9UXvvAFvfXWW922rwAAAMCgYNuUlAUAAOgiw7a75/mj999/X9OnT9ff/vY3HXfccZKkVatW6eyzz9bOnTtVWlraYp2qqioVFxdrxYoV+uIXvyhJ+uCDDzRt2jStW7dOJ5xwgl544QV9/vOf1+7duzV8+HBJ0s9+9jN9+9vf1t69e+X3+1O25/DDD9eCBQuSgrdtqa6uVn5+vqqqqpSXl3coX0G/YVmWKioqNGzYMJkmFS2AQ0VfAtKDvgSkR3f1pS2nnab8c87RsKuvTts2gb6OYxOQHvQlDHSdiSd6u6sR69atU0FBgRuQlaTZs2fLNE2tX79e5513Xot1Nm7cqHA4rNmzZ7vTpk6dqjFjxrhB2XXr1unII490A7KSNGfOHF111VXatGmTZsyY0WK7lmWppqZGRUVFrbY3GAwqGAy676urq911Lcvq3M73M5ZlybbtAb+fQHejLwHpQV8C0qPb+pItiT6KQYZjE5Ae9CUMdJ353e62oGx5ebmGDRuW/GFer4qKilReXt7qOn6/XwUFBUnThw8f7q5TXl6eFJCNz4/PS+Xuu+9WbW2tvvSlL7Xa3jvvvFO33npri+l79+5VY2Njq+sNBJZlqaqqSrZtc6cK6AL6EpAe9CUgPbqrL0WjUdXV18uqqEjbNoG+jmMTkB70JQx0NTU1HV6200HZ73znO/r+97/f5jLvv/9+ZzfbbVasWKFbb71VTz/9dIsgcaIbb7xRS5cudd9XV1dr9OjR7mBhA5llWTIMQ8XFxfxRBLqAvgSkB30JSI/u6ks1pqns7BwNbePcGhhoODYB6UFfwkCXkZHR4WU7HZS99tprddlll7W5zIQJE1RSUqKKZnfPI5GIDhw4oJKSkpTrlZSUKBQKqbKyMilbds+ePe46JSUl2rBhQ9J6e/bsceclWrlypb761a/qiSeeSCqJkEogEFAgEGgx3TTNQfGHwjCMQbOvQHeiLwHpQV8C0qNb+pJtu9sFBhOOTUB60JcwkHXm97rTQdni4mIVFxe3u1xZWZkqKyu1ceNGzZw5U5L00ksvybIszZo1K+U6M2fOlM/n05o1azR//nxJ0ubNm7Vjxw6VlZW5273jjjvcwtCStHr1auXl5Wn69Onuth5//HF95Stf0cqVKzVv3rzO7iYAAACAVDymbCva260AAADo17rttsS0adM0d+5cXXHFFdqwYYNee+01LVmyRBdeeKFKS0slSbt27dLUqVPdzNf8/HxdfvnlWrp0qV5++WVt3LhRX/7yl1VWVqYTTjhBknTmmWdq+vTpuuSSS/TOO+/oxRdf1E033aTFixe7ma4rVqzQokWLdM8992jWrFkqLy9XeXm5qqqqumt3AQAAgEHB8Plkh8O93QwAAIB+rVtzxR977DFNnTpVp59+us4++2yddNJJ+vnPf+7OD4fD2rx5s+rr691p9957rz7/+c9r/vz5+uxnP6uSkhL9/ve/d+d7PB4999xz8ng8Kisr07//+79r0aJFuu2229xlfv7znysSiWjx4sUaMWKE++/qq6/uzt0FAAAABjyCsgAAAF3X6fIFnVFUVKQVK1a0On/cuHGybTtpWkZGhpYvX67ly5e3ut7YsWP1/PPPtzp/7dq1nW4rAAAAgPYZfr/sUKi3mwEAANCvUVUZAAAAQIeZPj+ZsgAAAF1EUBYAAABAhxk+n+wQQVkAAICu6NbyBQAAAAAGlsJLLpGZEejtZgAAAPRrBGUBAAAAdFjenDN7uwkAAAD9HuULAAAAAAAAAKAHEZQFAAAAAAAAgB5EUBYAAAAAAAAAehBBWQAAAAAAAADoQQRlAQAAAAAAAKAHEZQFAAAAAAAAgB5EUBYAAAAAAAAAepC3txvQV9m2LUmqrq7u5ZZ0P8uyVFNTo4yMDJkmcXrgUNGXgPSgLwHpQV8C0of+BKQHfQkDXTyOGI8rtoWgbCtqamokSaNHj+7llgAAAAAAAADoL2pqapSfn9/mMobdkdDtIGRZlnbv3q3c3FwZhtHbzelW1dXVGj16tD755BPl5eX1dnOAfou+BKQHfQlID/oSkD70JyA96EsY6GzbVk1NjUpLS9vNBidTthWmaWrUqFG93YwelZeXxx9FIA3oS0B60JeA9KAvAelDfwLSg76Egay9DNk4CngAAAAAAAAAQA8iKAsAAAAAAAAAPYigLBQIBLRs2TIFAoHebgrQr9GXgPSgLwHpQV8C0of+BKQHfQlowkBfAAAAAAAAANCDyJQFAAAAAAAAgB5EUBYAAAAAAAAAehBBWQAAAAAAAADoQQRlB6jly5dr3LhxysjI0KxZs7Rhw4Y2l3/iiSc0depUZWRk6Mgjj9Tzzz+fNN+2bd18880aMWKEMjMzNXv2bG3ZsqU7dwHoE9Ldly677DIZhpH0b+7cud25C0Cf0Jm+tGnTJs2fP1/jxo2TYRj68Y9/3OVtAgNFuvvSLbfc0uK4NHXq1G7cA6Bv6Exfeuihh3TyySersLBQhYWFmj17dovluV7CYJXuvsT1EgYTgrID0P/93/9p6dKlWrZsmf7+97/r6KOP1pw5c1RRUZFy+ddff10XXXSRLr/8cr311ls699xzde655+qf//ynu8wPfvAD3X///frZz36m9evXKzs7W3PmzFFjY2NP7RbQ47qjL0nS3Llz9emnn7r/Hn/88Z7YHaDXdLYv1dfXa8KECbrrrrtUUlKSlm0CA0F39CVJOvzww5OOS3/961+7axeAPqGzfWnt2rW66KKL9PLLL2vdunUaPXq0zjzzTO3atctdhuslDEbd0ZckrpcwiNgYcI4//nh78eLF7vtoNGqXlpbad955Z8rlv/SlL9nz5s1LmjZr1iz7a1/7mm3btm1Zll1SUmL/8Ic/dOdXVlbagUDAfvzxx7thD4C+Id19ybZt+9JLL7XPOeecbmkv0Fd1ti8lGjt2rH3vvfemdZtAf9UdfWnZsmX20UcfncZWAn1fV48hkUjEzs3NtR999FHbtrlewuCV7r5k21wvYXAhU3aACYVC2rhxo2bPnu1OM01Ts2fP1rp161Kus27duqTlJWnOnDnu8tu2bVN5eXnSMvn5+Zo1a1ar2wT6u+7oS3Fr167VsGHDNGXKFF111VXav39/+ncA6CMOpS/1xjaBvq47f++3bNmi0tJSTZgwQQsXLtSOHTu62lygz0pHX6qvr1c4HFZRUZEkrpcwOHVHX4rjegmDBUHZAWbfvn2KRqMaPnx40vThw4ervLw85Trl5eVtLh//vzPbBPq77uhLkvMozv/+7/9qzZo1+v73v69XXnlFZ511lqLRaPp3AugDDqUv9cY2gb6uu37vZ82apUceeUSrVq3Sgw8+qG3btunkk09WTU1NV5sM9Enp6Evf/va3VVpa6gajuF7CYNQdfUniegmDi7e3GwAAg8mFF17ovj7yyCN11FFHaeLEiVq7dq1OP/30XmwZAGAwOuuss9zXRx11lGbNmqWxY8fqt7/9rS6//PJebBnQN911111auXKl1q5dq4yMjN5uDtBvtdaXuF7CYEKm7AAzdOhQeTwe7dmzJ2n6nj17Wh3goaSkpM3l4/93ZptAf9cdfSmVCRMmaOjQofrwww+73migDzqUvtQb2wT6up76vS8oKNDkyZM5LmHA6kpfuvvuu3XXXXfpT3/6k4466ih3OtdLGIy6oy+lwvUSBjKCsgOM3+/XzJkztWbNGneaZVlas2aNysrKUq5TVlaWtLwkrV692l1+/PjxKikpSVqmurpa69evb3WbQH/XHX0plZ07d2r//v0aMWJEehoO9DGH0pd6Y5tAX9dTv/e1tbXaunUrxyUMWIfal37wgx/o9ttv16pVq3TcccclzeN6CYNRd/SlVLhewoDW2yONIf1WrlxpBwIB+5FHHrHfe+89+8orr7QLCgrs8vJy27Zt+5JLLrG/853vuMu/9tprttfrte+++277/ffft5ctW2b7fD773XffdZe566677IKCAvvpp5+2//GPf9jnnHOOPX78eLuhoaHH9w/oKenuSzU1NfZ1111nr1u3zt62bZv95z//2T722GPtSZMm2Y2Njb2yj0BP6GxfCgaD9ltvvWW/9dZb9ogRI+zrrrvOfuutt+wtW7Z0eJvAQNQdfenaa6+1165da2/bts1+7bXX7NmzZ9tDhw61Kyoqenz/gJ7S2b5011132X6/337yySftTz/91P1XU1OTtAzXSxhs0t2XuF7CYENQdoD6yU9+Yo8ZM8b2+/328ccfb7/xxhvuvFNOOcW+9NJLk5b/7W9/a0+ePNn2+/324Ycfbv/xj39Mmm9Zlv3d737XHj58uB0IBOzTTz/d3rx5c0/sCtCr0tmX6uvr7TPPPNMuLi62fT6fPXbsWPuKK64giIRBoTN9adu2bbakFv9OOeWUDm8TGKjS3ZcWLFhgjxgxwvb7/fbIkSPtBQsW2B9++GEP7hHQOzrTl8aOHZuyLy1btsxdhuslDFbp7EtcL2GwMWzbtns2NxcAAAAAAAAABi9qygIAAAAAAABADyIoCwAAAAAAAAA9iKAsAAAAAAAAAPQggrIAAAAAAAAA0IMIygIAAAAAAABADyIoCwAAAAAAAAA9iKAsAAAAAAAAAPQggrIAAAAAAAAA0IMIygIAAGBAW7t2rQzDUGVlZa98/po1azRt2jRFo9F2l121apWOOeYYWZbVAy0DAABAbyEoCwAAgAHjc5/7nK655pqkaSeeeKI+/fRT5efn90qbbrjhBt10003yeDztLjt37lz5fD499thjPdAyAAAA9BaCsgAAABjQ/H6/SkpKZBhGj3/2X//6V23dulXz58/v8DqXXXaZ7r///m5sFQAAAHobQVkAAAAMCJdddpleeeUV3XfffTIMQ4ZhaPv27S3KFzzyyCMqKCjQc889pylTpigrK0tf/OIXVV9fr0cffVTjxo1TYWGhvvnNbyaVHAgGg7ruuus0cuRIZWdna9asWVq7dm2bbVq5cqXOOOMMZWRkuNPeeecdnXrqqcrNzVVeXp5mzpypN998053/hS98QW+++aa2bt2a1u8HAAAAfYe3txsAAAAApMN9992nf/3rXzriiCN02223SZKKi4u1ffv2FsvW19fr/vvv18qVK1VTU6Pzzz9f5513ngoKCvT888/ro48+0vz58/Vv//ZvWrBggSRpyZIleu+997Ry5UqVlpbqD3/4g+bOnat3331XkyZNStmmv/zlL7r44ouTpi1cuFAzZszQgw8+KI/Ho7fffls+n8+dP2bMGA0fPlx/+ctfNHHixDR9OwAAAOhLCMoCAABgQMjPz5ff71dWVpZKSkraXDYcDuvBBx90g55f/OIX9etf/1p79uxRTk6Opk+frlNPPVUvv/yyFixYoB07dujhhx/Wjh07VFpaKkm67rrrtGrVKj388MP67//+75Sf8/HHH7vLx+3YsUPXX3+9pk6dKkkpA7qlpaX6+OOPO/0dAAAAoH8gKAsAAIBBJysrKykLdfjw4Ro3bpxycnKSplVUVEiS3n33XUWjUU2ePDlpO8FgUEOGDGn1cxoaGpJKF0jS0qVL9dWvflW//vWvNXv2bF1wwQUtMmIzMzNVX19/yPsHAACAvo2gLAAAAAadxHIBkmQYRspplmVJkmpra+XxeLRx40Z5PJ6k5RIDuc0NHTpUBw8eTJp2yy236OKLL9Yf//hHvfDCC1q2bJlWrlyp8847z13mwIEDKi4uPqR9AwAAQN9HUBYAAAADht/vTxqcK11mzJihaDSqiooKnXzyyZ1a77333msxffLkyZo8ebK+9a1v6aKLLtLDDz/sBmUbGxu1detWzZgxI23tBwAAQN9i9nYDAAAAgHQZN26c1q9fr+3bt2vfvn1upmtXTZ48WQsXLtSiRYv0+9//Xtu2bdOGDRt055136o9//GOr682ZM0d//etf3fcNDQ1asmSJ1q5dq48//livvfaa/va3v2natGnuMm+88YYCgYDKysrS0nYAAAD0PQRlAQAAMGBcd9118ng8mj59uoqLi7Vjx460bfvhhx/WokWLdO2112rKlCk699xz9be//U1jxoxpdZ2FCxdq06ZN2rx5syTJ4/Fo//79WrRokSZPnqwvfelLOuuss3Trrbe66zz++ONauHChsrKy0tZ2AAAA9C2Gbdt2bzcCAAAAGKiuv/56VVdX63/+53/aXXbfvn2aMmWK3nzzTY0fP74HWgcAAIDeQKYsAAAA0I3+67/+S2PHju1QKYXt27frgQceICALAAAwwJEpCwAAAAAAAAA9iExZAAAAAAAAAOhBBGUBAAAAAAAAoAcRlAUAAAAAAACAHkRQFgAAAAAAAAB6EEFZAAAAAAAAAOhBBGUBAAAAAAAAoAcRlAUAAAAAAACAHkRQFgAAAAAAAAB6EEFZAAAAAAAAAOhBBGUBAAAAAAAAoAf9f2C0D3c4jbsYAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAABI0AAAEpCAYAAAAefdJrAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjksIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvJkbTWQAAAAlwSFlzAAAPYQAAD2EBqD+naQABAABJREFUeJzsnXd4FEUfgN+9np4ACaGH3jtIU5oooKioiGChKIoF/VCxN1CKoigqigUERRQFFSwIiCC9Su+9k0p6cnXn+2PvLjku5QLBIMz7PPfc3ezszOzu7O7Mb35FEUIIJBKJRCKRSCQSiUQikUgkknzoyroBEolEIpFIJBKJRCKRSCSSyw8pNJJIJBKJRCKRSCQSiUQikfghhUYSiUQikUgkEolEIpFIJBI/pNBIIpFIJBKJRCKRSCQSiUTihxQaSSQSiUQikUgkEolEIpFI/JBCI4lEIpFIJBKJRCKRSCQSiR9SaCSRSCQSiUQikUgkEolEIvFDCo0kEolEIpFIJBKJRCKRSCR+SKGRRCKRSCQSiUQikUgkEonEDyk0kkgkEkmZoCgKo0ePLjbf6NGjURTl0jdIUiQbN27EZDJx/Phxb1rXrl3p2rXrJa1XURRGjBhxQfs6nU6ee+45qlWrhk6no2/fvqXbOMllSVZWFsOGDSM2NhZFURg5cmRZN0lShsycORNFUdi8eXNZN+WCuBTP2WPHjqEoCjNnzizVcj288MILtGvX7pKULZFI/n2k0EgikfwnURSl2E8gAgnJ1cnChQsL7B85OTmMHj2av//++19v0+XOyy+/zMCBA6lRo0ZZNyVgvvzyS9555x369evHV199xVNPPVXWTbrq2LNnD6NHj+bYsWMB5V+5ciW33nor1apVw2KxEBsbS69evVizZk3AdY4fP56ZM2fy6KOPMmvWLO6///4LbP3liaqqREdHM3HixELznD17lhdeeIFu3boRFhaGoigFPtdycnL4+OOPufHGG6lUqRJhYWG0bNmSqVOn4nK5Cqx74sSJ1KxZE4vFQrNmzfjuu+9K8/AkVwAjR45k+/bt/PLLL2XdFIlEUgoYyroBEolEciHMmjWr0G2jR4/m8OHDcpXrMic3NxeDoWxeQwsXLuTjjz/2Exzl5OQwZswYgEuuQfNfYtu2bSxdupS1a9eWdVNKxLJly6hSpQrvv/9+WTflqmXPnj2MGTOGrl27EhcXV2z+AwcOoNPpeOSRR4iNjSU1NZVvvvmGzp078/vvv9OrV69iy1i2bBnt27fn9ddfL4UjuPzYuHEjycnJ3HzzzYXm2b9/P2+//TZ169aladOmrFu3rsB8R44c4YknnuD666/n6aefJjw8nMWLF/PYY4+xfv16vvrqK5/8L7/8Mm+99RYPPfQQbdu2ZcGCBdxzzz0oisKAAQNK9Tgll44aNWqQm5uL0Wi8JOXHxsZy22238e6773LrrbdekjokEsm/hxQaSSSS/yT33XdfgenTpk3j8OHDPPHEE/Tu3ftfbtWVRXZ2NiEhIZesfIvFcsnKvty41OfyUjNjxgyqV69O+/bty7opJSIxMZHIyMhSK09VVex2+1XVd/9thg0bxrBhw3zSHnvsMWrVqsXkyZMDEholJibSqFGjYvNZrVZMJhM63X9L8X7hwoXUqFGDxo0bF5qndevWpKSkUK5cOebNm8ddd91VYL7Y2Fh27tzpU9bw4cN54IEHmDFjBq+++ip16tQB4PTp00yaNInHH3+cKVOmANr16tKlC88++yx33XUXer2+FI9UUto4nU5UVcVkMl3y51j//v256667OHLkCLVq1bqkdUkkkkvLf+stKZFIJEWwe/dunnzySVq2bMk777zjsy07O5tnnnmGatWqYTabqV+/Pu+++y5CCJ98TqeTN998k9q1a2M2m4mLi+Oll17CZrP55IuLi6NPnz78/ffftGnThqCgIJo2bepV///pp59o2rQpFouF1q1bs3Xr1mLb7/G7sGbNGp5++mmio6MJCQnh9ttvJykpyS//H3/8wXXXXUdISAhhYWHcfPPN7N692yfPjh07GDJkCLVq1fKaejzwwAOkpKT45PP4DdqzZw/33HMPUVFRXHvttcW2dfXq1Tz55JNER0cTGRnJ8OHDsdvtpKWlMWjQIKKiooiKiuK5557zO9cFmRCuXr2atm3bYrFYqF27Np999lmx583D33//XaAJxvm+G4YMGcLHH3/sbYPnc+zYMaKjowEYM2ZMgWaO+/bto1+/fpQrVw6LxUKbNm381O8952bFihU89thjxMTEULVqVQCOHz/OY489Rv369QkKCqJ8+fLcddddfqY7F9IXunTpQlhYGOHh4bRt25Zvv/3WJ8+GDRvo1asXERERBAcH06VLl4BNfubPn0/37t0D8i2VmJjIgw8+SMWKFbFYLDRv3txPWwECvycLYuzYseh0Oj766KMCt3uu+fLly9m9e7f3Wnr6RqB1e/wpzZ49m8aNG2M2m1m0aFGh7dq8eTM9e/akQoUKBAUFUbNmTR544AGfPKqqMnnyZBo3bozFYqFixYoMHz6c1NRUn3wLFizg5ptvpnLlypjNZmrXrs2bb77pZzLUtWtXmjRpwo4dO+jSpQvBwcHUqVOHefPmAbBixQratWtHUFAQ9evXZ+nSpT77Z2ZmMnLkSOLi4jCbzcTExHDDDTewZcuWwi8AgfXlmTNneoUV3bp187sOgRIcHEx0dDRpaWlF5vM8A44ePcrvv//uc297ts2ZM4dXXnmFKlWqEBwcTEZGBufOnWPUqFE0bdqU0NBQwsPD6d27N9u3by+w/B9++IExY8ZQpUoVwsLC6NevH+np6dhsNkaOHElMTAyhoaEMHTrU790B8M0339C6dWuCgoIoV64cAwYM4OTJkwGfj99//71ILSOAsLAwypUrV2xZFSpUKFD4dPvttwOwd+9eb9qCBQtwOBw89thj3jRFUXj00Uc5depUodpMHoYMGUJoaCgnTpygT58+hIaGUqVKFe/zeOfOnXTv3p2QkBBq1Kjh9wxzOByMGTOGunXrYrFYKF++PNdeey1//vlnsccJmibp8OHDKV++POHh4QwaNMjvvgP45JNPvPd75cqVefzxx/36XlxcHEOGDPHb93z/Q/n7zLhx46hatSoWi4Xrr7+eQ4cO+e3/+eefU7t2bYKCgrjmmmtYtWqVXx673c5rr71G69atiYiIICQkhOuuu47ly5f75PM8B999910mT57sHdfs2bOnUJ9GgbzjAr0OPXr0ALR+I5FI/ttITSOJRHJFkJOTQ//+/dHr9cyZMwez2ezdJoTg1ltvZfny5Tz44IO0aNGCxYsX8+yzz3L69Gkf05Vhw4bx1Vdf0a9fP5555hk2bNjAhAkT2Lt3Lz///LNPnYcOHeKee+5h+PDh3Hfffbz77rvccsstfPrpp7z00kvegfWECRPo378/+/fvD2hF+4knniAqKorXX3+dY8eOMXnyZEaMGMH333/vzTNr1iwGDx5Mz549efvtt8nJyWHq1Klce+21bN261WsG8ueff3LkyBGGDh1KbGwsu3fv5vPPP2f37t2sX7/eTwhw1113UbduXcaPHx/Q5P2JJ54gNjaWMWPGsH79ej7//HMiIyNZu3Yt1atXZ/z48SxcuJB33nmHJk2aMGjQoELL2rlzJzfeeCPR0dGMHj0ap9PJ66+/TsWKFYttR0kYPnw4Z86c4c8///Qxc4yOjmbq1Kk8+uij3H777dxxxx0ANGvWDNCEkp06daJKlSq88MILhISE8MMPP9C3b19+/PFH7yTLw2OPPUZ0dDSvvfYa2dnZAGzatIm1a9cyYMAAqlatyrFjx5g6dSpdu3Zlz549BAcH+5QRSF+YOXMmDzzwAI0bN+bFF18kMjKSrVu3smjRIu655x5AM9fp3bs3rVu35vXXX0en0zFjxgy6d+/OqlWruOaaawo9X6dPn+bEiRO0atWq2HObm5tL165dOXToECNGjKBmzZrMnTuXIUOGkJaWxv/+9z+gZPfk+bzyyiuMHz+ezz77jIceeqjAPNHR0cyaNYtx48aRlZXFhAkTAGjYsGGJ6162bBk//PADI0aMoEKFCoWaWCUmJnr77wsvvEBkZCTHjh3jp59+8sk3fPhwZs6cydChQ3nyySc5evQoU6ZMYevWraxZs8ZrLjJz5kxCQ0N5+umnCQ0NZdmyZbz22mtkZGT4CcVTU1Pp06cPAwYM4K677mLq1KkMGDCA2bNnM3LkSB555BHuuecer3+nkydPEhYWBsAjjzzCvHnzGDFiBI0aNSIlJYXVq1ezd+/eIq95IH25c+fOPPnkk3z44Ye89NJLNGzY0HsdiiMjIwO73U5ycjJff/01u3bt4qWXXipyn4YNGzJr1iyeeuopqlatyjPPPANo/cEjzHrzzTcxmUyMGjUKm82GyWRiz549zJ8/n7vuuouaNWuSkJDAZ599RpcuXdizZw+VK1f2qWfChAkEBQXxwgsvcOjQIT766COMRiM6nY7U1FRGjx7N+vXrmTlzJjVr1uS1117z7jtu3DheffVV+vfvz7Bhw0hKSuKjjz6ic+fObN26tVjNuPj4eLZu3cobb7xR7Dm8GOLj4wFNqORh69athISE+F0/z/Nj69atRS42ALhcLnr37k3nzp2ZOHEis2fPZsSIEYSEhPDyyy9z7733cscdd/Dpp58yaNAgOnToQM2aNQFtcWPChAkMGzaMa665hoyMDDZv3syWLVu44YYbij2mESNGEBkZyejRo9m/fz9Tp07l+PHjXsGOp44xY8bQo0cPHn30UW++TZs2+dyfJeWtt95Cp9MxatQo0tPTmThxIvfeey8bNmzw5pk+fTrDhw+nY8eOjBw5kiNHjnDrrbdSrlw5qlWr5s2XkZHBtGnTGDhwIA899BCZmZlMnz6dnj17snHjRlq0aOFT94wZM7BarTz88MOYzWbKlSuHqqp+bQz0HRfodYiIiKB27dqsWbNG+pOTSP7rCIlEIrkCeOCBBwQgvvrqK79t8+fPF4AYO3asT3q/fv2Eoiji0KFDQgghtm3bJgAxbNgwn3yjRo0SgFi2bJk3rUaNGgIQa9eu9aYtXrxYACIoKEgcP37cm/7ZZ58JQCxfvrzIY5gxY4YARI8ePYSqqt70p556Suj1epGWliaEECIzM1NERkaKhx56yGf/+Ph4ERER4ZOek5PjV893330nALFy5Upv2uuvvy4AMXDgwCLbeH5be/bs6dPWDh06CEVRxCOPPOJNczqdomrVqqJLly4+ZQDi9ddf9/7v27evsFgsPuduz549Qq/Xi0BeV8uXLy/wPB89elQAYsaMGd60xx9/vMAyk5KS/Nrl4frrrxdNmzYVVqvVm6aqqujYsaOoW7euN81zbq699lrhdDp9yijoeqxbt04A4uuvv/Yro7i+kJaWJsLCwkS7du1Ebm6uT7me/VRVFXXr1vW7Vjk5OaJmzZrihhtu8GtTfpYuXSoA8euvv/pt69Kli891nTx5sgDEN998402z2+2iQ4cOIjQ0VGRkZAghAr8nhdD6yeOPPy6EEOKZZ54ROp1OzJw5s8g2529f48aNfdJKWrdOpxO7d+8utq6ff/5ZAGLTpk2F5lm1apUAxOzZs33SFy1a5JdeUF8ZPny4CA4O9umDXbp0EYD49ttvvWn79u3ztn39+vXedM8zKv+9EBER4T2/JSHQvjx37tyAnn/n07NnTwEIQJhMJjF8+HC/Pl4YNWrUEDfffLNPmuf5UKtWLb+2W61W4XK5fNKOHj0qzGazeOONN/zKaNKkibDb7d70gQMHCkVRRO/evX3K6NChg6hRo4b3/7Fjx4Rerxfjxo3zybdz505hMBj80gti+vTpIigoqMDzXxglvQY2m000atRI1KxZUzgcDm/6zTffLGrVquWXPzs7WwDihRdeKLLcwYMHC0CMHz/em5aamiqCgoKEoihizpw53nRPH87/LG7evLnfdQ0Ez/O0devWPtdt4sSJAhALFiwQQgiRmJgoTCaTuPHGG336w5QpUwQgvvzyS29ajRo1xODBg/3qOv+Z6OkzDRs2FDabzZv+wQcfCEDs3LlTCKE9J2NiYkSLFi188n3++ecC8CnT6XT65BFCO48VK1YUDzzwgDfN8+4LDw8XiYmJPvkLei8G+o4ryXW48cYbRcOGDQPKK5FILl+keZpEIvnP8+233/Lll19y//33F6jJsnDhQvR6PU8++aRP+jPPPIMQgj/++MObD+Dpp5/2yweaSUB+GjVqRIcOHbz/PY63u3fvTvXq1f3Sjxw5EtDxPPzwwz4aQNdddx0ul8sb6vzPP/8kLS2NgQMHkpyc7P3o9XratWvno6IeFBTk/W21WklOTvb6pSnI/OSRRx4JqI0eHnzwQZ+2tmvXDiEEDz74oDdNr9fTpk2bIo/f5XKxePFi+vbt63PuGjZsSM+ePUvUpkvBuXPnWLZsGf379yczM9N7zlNSUujZsycHDx7k9OnTPvs89NBDfv498l8Ph8NBSkoKderUITIyssDrEUhfyMzM5IUXXvDzT+HZb9u2bRw8eJB77rmHlJQUb9uzs7O5/vrrWblyZYGrzh48poxRUVHFnqeFCxcSGxvLwIEDvWlGo5Enn3ySrKwsVqxY4c0XyD3pQQjBiBEj+OCDD/jmm28YPHhwsW0pqo0lqbtLly4B+cfxaIj89ttvOByOAvPMnTuXiIgIbrjhBp97t3Xr1oSGhhZ673r63HXXXUdOTg779u3zKTc0NNTHCXH9+vWJjIykYcOGPgEBCnoWRUZGsmHDBs6cOVPsMeanpH25pLz11lssWbKE6dOn0759e+x2O06n86LLHTx4sE/bAcxms1cL1OVykZKSQmhoKPXr1y/wWAYNGuSjceJ57p1vitiuXTtOnjzpbfdPP/2Eqqr079/f5/rHxsZSt25dP/Oigli4cCHdunXzO4bSZMSIEezZs4cpU6b4BCvIzc310eL14Hn25ObmBlR+fp9VkZGR1K9fn5CQEPr37+9N9/Th8/vq7t27OXjwYImPCbTnaf7r9uijj2IwGLzv/qVLl2K32xk5cqSPVvBDDz1EeHi43xigJAwdOhSTyeT9f9111wF59+LmzZtJTEzkkUce8ck3ZMgQIiIifMrS6/XePKqqcu7cOZxOJ23atCmwv955551e0+vCKMk7riTXISoqiuTk5GLzSSSSyxtpniaRSP7THDx4kEceeYR69erxySefFJjn+PHjVK5c2WuO4cGjYu+ZgB8/fhydTud1+ukhNjaWyMhIbz4P+YUbgHdgl1+NPH96Qb4TCuL8cj2Tdc/+noFa9+7dC9w/PDzc+/vcuXOMGTOGOXPmkJiY6JMvPT3db1+PGUCglOQcFHX8SUlJ5ObmUrduXb9t9evX9w7qQTsmu93u/R8UFOQ3qC5tDh06hBCCV199lVdffbXAPImJiVSpUsX7v6BzmZuby4QJE5gxYwanT5/2MQEs6HoU1xcOHz4MQJMmTQptu6e/FCVoSU9PL1YoJAIwVzx+/Dh169b1M8Ms6F4L5J708PXXX5OVlcXUqVN9BFIXQknrDvSe6NKlC3feeSdjxozh/fffp2vXrvTt25d77rnHO9E+ePAg6enpxMTEFFhG/nt09+7dvPLKKyxbtoyMjAyffOf3lapVq/qZmkZERAT0LJo4cSKDBw+mWrVqtG7dmptuuolBgwYV67i2pH25pOQ3sbnvvvto1aoVQ4YM8fpqutDnQEHXU1VVPvjgAz755BOOHj3q4zeqfPnyfvlL8txTVZX09HTKly/PwYMHEUIU+JwDijV9cjgc/Pnnn15zy0vBO++8wxdffMGbb77JTTfd5LMtKCioQB9NVqvVu704LBaLnwAjIiKi0D6cv6++8cYb3HbbbdSrV48mTZrQq1cv7r//fq8JcXGcf95DQ0OpVKmS13TRc+/Xr1/fJ5/JZKJWrVp+z4aSUNyz3FP2+W00Go0F3otfffUVkyZNYt++fT5C6oL6dyDPsJK840pyHYQQAfnCk0gklzdSaCSRSP6z2Gw27r77bux2O3PmzCE0NLRUyg10gFNYlJjC0gOZdAeyv0crZNasWcTGxvrly78y3L9/f9auXcuzzz5LixYtCA0NRVVVevXqVaB2SUlXr0tyDgI9/uK44447vBoroAlDPI6jC+J8x8EXgudcjRo1qlDNp/OFjQWdyyeeeIIZM2YwcuRIOnToQEREhDdUdUHX42L7Uv62v/POO36+LjwUde94Js2BCj0vBZ06dWLbtm1MmTKF/v37B+Tgt7QI9J5QFIV58+axfv16fv31VxYvXswDDzzApEmTWL9+vffei4mJYfbs2QWW4ZlMp6Wl0aVLF8LDw3njjTeoXbs2FouFLVu28Pzzz/v1lYt5FvXv35/rrruOn3/+mSVLlvDOO+/w9ttv89NPPxUZgbKkffliMJlM3Hrrrbz11lvk5uYSFBRU6HOgOAq6nuPHj+fVV1/lgQce4M0336RcuXLodDpGjhxZovsykGe3oij88ccfBeYt7h22evVqMjIy/IQ5pcXMmTN5/vnneeSRR3jllVf8tleqVInly5f7CQLOnj0L4Of7qSAupq927tyZw4cPs2DBApYsWcK0adN4//33+fTTT/0i7l1qinrfFHQspfEs9/DNN98wZMgQ+vbty7PPPktMTAx6vZ4JEyZ4FxLyE8gzrCTvuJJch9TUVB+/WBKJ5L+JFBpJJJL/LKNGjWLr1q188MEHtGzZstB8NWrUYOnSpWRmZvpoF3hMPGrUqOH9VlWVgwcP+jj6TEhIIC0tzZuvrKlduzYAMTEx3ugkBZGamspff/3FmDFjfByxXqhq/6UkOjqaoKCgAtu2f/9+n/+TJk3yEWB4Jiqeldvzo9wUtDpc2IC/sHTPSq/RaCzynBfHvHnzGDx4MJMmTfKmWa3WYqNCFYanL+zatctPaHV+nvDw8Atqe4MGDQA4evRosXlr1KjBjh07UFXVR9uooHstkHvSQ506dZg4cSJdu3alV69e/PXXX36aQoFS0rpLSvv27Wnfvj3jxo3j22+/5d5772XOnDkMGzaM2rVrs3TpUjp16lTkRO7vv/8mJSWFn376ic6dO3vTA7kGF0KlSpV47LHHeOyxx0hMTKRVq1aMGzeuSKFRoH25tLQMcnNzEUKQmZlJUFBQoc+BC2HevHl069aN6dOn+6SnpaWV6oS3du3aCCGoWbMm9erVK/H+v//+O40aNSrUGfvFsGDBAoYNG8Ydd9zhjWZ2Pi1atGDatGns3bvXx2TT48y5MKF0aVKuXDmGDh3K0KFDycrKonPnzowePTogodHBgwfp1q2b939WVhZnz571CuE89/7+/ft9tHvsdjtHjx71eX5GRUUV+Nw+fvz4BYWX99R98OBBHy1ih8PB0aNHad68uTdt3rx51KpVi59++snn/nr99ddLXK+Hkr7jAr0O57ddIpH8N5E+jSQSyX+Sn3/+mSlTpnDrrbf6+SY5n5tuugmXy8WUKVN80t9//30URfFOjDwDx8mTJ/vke++99wCKDXH8b9GzZ0/Cw8MZP358gb5TPCHZPSub569knn98lwN6vZ6ePXsyf/58Tpw44U3fu3cvixcv9snbunVrevTo4f14Ji81atRAr9ezcuVKn/wFmS2GhIQA/gImT/Sy89NjYmLo2rUrn332mXdVPT+ecx7IcZ5/PT766KML1oa68cYbCQsLY8KECV4TEQ+eelq3bk3t2rV59913ycrKKnHbq1SpQrVq1di8eXOx7bnpppuIj4/3ie7mdDr56KOPCA0NpUuXLt58gdyT+WnWrBkLFy5k79693HLLLQH7TymojSWtOxBSU1P9rq1nEu0x6enfvz8ul4s333zTb3+n0+ntdwXdu3a7vVAT3AvF5XL5mZLFxMRQuXLlAs2Q8hNoXy7sXiuM881oPfv++OOPVKtWzWvaV9hz4EIo6Fjmzp3r56fsYrnjjjvQ6/WMGTPGrz4hhNd/WGEsXLjwkryHVq5cyYABA+jcuTOzZ88uNMrnbbfdhtFo9OmHQgg+/fRTqlSpQseOHUu9bfk5//yEhoZSp06dYvuqh88//9znnTl16lScTqf3nu/Rowcmk4kPP/zQ5/pMnz6d9PR0n3Nfu3Zt1q9f72Mi+dtvv3Hy5MkLOrY2bdoQHR3Np59+6lPmzJkz/e6dgp4PGzZsYN26dRdUN5TsHRfodUhPT+fw4cOXvF9IJJJLj9Q0kkgk/znOnj3Lgw8+iF6v5/rrr+ebb74pMF/t2rXp0KEDt9xyC926dePll1/m2LFjNG/enCVLlrBgwQJGjhzp1cRo3rw5gwcP5vPPP/eah2zcuJGvvvqKvn37+qxQliXh4eFMnTqV+++/n1atWjFgwACio6M5ceIEv//+O506dWLKlCmEh4d7wxo7HA6qVKnCkiVLLpm2wsUyZswYFi1axHXXXcdjjz3mFTY0btyYHTt2FLt/REQEd911Fx999BGKolC7dm1+++23AiehrVu3BuDJJ5+kZ8+e6PV6BgwYQFBQEI0aNeL777+nXr16lCtXjiZNmtCkSRM+/vhjrr32Wpo2bcpDDz1ErVq1SEhIYN26dZw6dYrt27cX28Y+ffowa9YsIiIiaNSoEevWrWPp0qUF+k0JhPDwcN5//32GDRtG27Ztueeee4iKimL79u3k5OTw1VdfodPpmDZtGr1796Zx48YMHTqUKlWqcPr0aZYvX054eDi//vprkfXcdttt/Pzzz8X6p3j44Yf57LPPGDJkCP/88w9xcXHMmzePNWvWMHnyZK9mT6D35Pm0b9+eBQsWcNNNN9GvXz/mz59f4hDYF1p3cXz11Vd88skn3H777dSuXZvMzEy++OILwsPDvQLpLl26MHz4cCZMmMC2bdu48cYbMRqNHDx4kLlz5/LBBx/Qr18/OnbsSFRUFIMHD+bJJ59EURRmzZpVaiaeHjIzM6latSr9+vWjefPmhIaGsnTpUjZt2uSjQVQQgfblFi1aoNfrefvtt0lPT8dsNtO9e/dC/Tr17t2bqlWr0q5dO2JiYjhx4gQzZszgzJkzPsLI0qRPnz688cYbDB06lI4dO7Jz505mz559QRojRVG7dm3Gjh3Liy++yLFjx+jbty9hYWEcPXqUn3/+mYcffphRo0YVuO/Ro0fZu3cvU6dODbi+sWPHApp/LNBMmlevXg3gNT87fvw4t956K4qi0K9fP+bOnetTRrNmzby+aqpWrcrIkSN55513cDgctG3blvnz57Nq1Spmz55dqAlWadGoUSO6du1K69atKVeuHJs3b2bevHmMGDEioP3tdjvXX389/fv3Z//+/XzyySdce+213HrrrYCm8friiy8yZswYevXqxa233urN17ZtW+677z5vWcOGDWPevHn06tWL/v37c/jwYb755psLfn4YjUbGjh3L8OHD6d69O3fffTdHjx5lxowZfv2wT58+/PTTT9x+++3cfPPNHD16lE8//ZRGjRoVuDAQKIG+4wK9DkuXLkUIwW233XbBbZJIJJcJlzAym0QikVwSPCFsi/vkD4ebmZkpnnrqKVG5cmVhNBpF3bp1xTvvvOMTglwIIRwOhxgzZoyoWbOmMBqNolq1auLFF1/0CUErRMEhnYXwDQ/uwRPa9p133inyuDxhgc8P2V1YKPnly5eLnj17ioiICGGxWETt2rXFkCFDxObNm715Tp06JW6//XYRGRkpIiIixF133SXOnDnjF8r49ddfF4BISkoqso3FtbWwcgYPHixCQkJ80s5vgxBCrFixQrRu3VqYTCZRq1Yt8emnn3rLDISkpCRx5513iuDgYBEVFSWGDx8udu3a5Rda2Ol0iieeeEJER0cLRVF8yl+7dq23Dee38fDhw2LQoEEiNjZWGI1GUaVKFdGnTx8xb968Ys+NEFpY5KFDh4oKFSqI0NBQ0bNnT7Fv3z6/8M0l7Qu//PKL6NixowgKChLh4eHimmuuEd99951Pnq1bt4o77rhDlC9fXpjNZlGjRg3Rv39/8ddffxV7Xrds2SIAsWrVKp/088NLCyFEQkKC9xhNJpNo2rSpz7n3EOg9WdA9tWDBAmEwGMTdd9/tFyr9/PY1bty4VOsujC1btoiBAweK6tWrC7PZLGJiYkSfPn187kcPn3/+uWjdurUICgoSYWFhomnTpuK5554TZ86c8eZZs2aNaN++vQgKChKVK1cWzz33nFi8eLHf9S/sGAN5RtlsNvHss8+K5s2bi7CwMBESEiKaN28uPvnkk2KPN9C+LIQQX3zxhahVq5bQ6/XFhn6fMmWKuPbaa0WFChWEwWAQ0dHR4pZbbhErV64stk1FHbvn3pk7d65ffqvVKp555hlRqVIlERQUJDp16iTWrVtXaPj088so6fPwxx9/FNdee60ICQkRISEhokGDBuLxxx8X+/fvL/SYpkyZIiIiIoTD4Qj0NBT5fjz/mAr7nP+MdrlcYvz48aJGjRrCZDKJxo0bi2+++Sag9hT0HhAi8D48duxYcc0114jIyEgRFBQkGjRoIMaNGyfsdnuR9Xquz4oVK8TDDz8soqKiRGhoqLj33ntFSkqKX/4pU6aIBg0aCKPRKCpWrCgeffRRkZqa6pdv0qRJokqVKsJsNotOnTqJzZs3B9xnCgp5L4QQn3zyiahZs6Ywm82iTZs2YuXKlX5lqqrqvQZms1m0bNlS/Pbbb2Lw4MGiRo0afnUUNPYorP5A3nGBXoe7775bXHvttX51SySS/x6KEKW8bCWRSCQSieSK4/rrr6dy5crMmjWrrJsikVx13HTTTYSGhvLDDz+UdVMkkmKJj4+nZs2azJkzR2oaSSRXANKnkUQikUgkkmIZP34833///UWFnZZIJBdG165deeqpp8q6GRJJQEyePJmmTZtKgZFEcoUgNY0kEolEIpFIJBKJRCKRSCR+SE0jiUQikUgkEolEIpFIJBKJH1JoJJFIJBKJRCKRSCQSiUQi8UMKjSQSiUQikUgkEolEIpFIJH5IoZFEIpFIJBKJRCKRSCQSicQPQ1k34HJFVVXOnDlDWFgYiqKUdXMkEolEIpFIJBKJRCKRSEoFIQSZmZlUrlwZna5wfSIpNCqEM2fOUK1atbJuhkQikUgkEolEIpFIJBLJJeHkyZNUrVq10O1SaFQIYWFhgHYCw8PDy7g1F4aqqiQlJREdHV2k5FAi8SD7jKSkyD4jKSmyz0hKiuwzkpIi+4ykpMg+IykpV0KfycjIoFq1al7ZR2FIoVEheEzSwsPD/9NCI6vVSnh4+H+2I0v+XWSfkZQU2WckJUX2GUlJkX1GUlJkn5GUFNlnJCXlSuozxbnj+W8fnUQikUgkEolEIpFIJBKJ5JIghUYSiUQikUgkEolEIpFIJBI/pNBIIpFIJBKJRCKRSCQSiUTih/RpJJFIJBKJRCKRSCQSiaTUcblcOByOsm5GqaOqKg6HA6vVetn6NDIajej1+osuRwqNJBKJRCKRSCQSiUQikZQaQgji4+NJS0sr66ZcEoQQqKpKZmZmsY6ky5LIyEhiY2Mvqo1SaCSRSCQSiUQikUgkEomk1PAIjGJiYggODr6sBSsXghACp9OJwWC4LI9NCEFOTg6JiYkAVKpU6YLLkkIjiUQikUgkEolEIpFIJKWCy+XyCozKly9f1s25JFzuQiOAoKAgABITE4mJiblgU7XL0/hOIrna+KwLZCWVdSskEolEIpFIJBKJ5KLw+DAKDg4u45ZIPNfgYvxKSaGRRFLWuJxwdhtkJZR1SyQSiUQikUgkEomkVLhcNXCuJkrjGkihkURS1uSmat/la5dtOyQSiUQikUgkEolEIsmHFBpJJGVN7jntOzO+bNshkUgkEolEIpFIJFcxQggefvhhypUrh6IobNu2rdh9FEVh/vz5ABw7dizg/f4rSEfYEklZE1lD+7Znl207JBKJRCKRSCQSieQqZtGiRcycOZO///6bWrVqUaFChbJuUpkjhUYSSVmjN4IhCBBl3RKJRCKRSCQSiUQiuWo5fPgwlSpVomPHjmXdlMsGaZ4mkZQ1h5eBMxeEWtYtkUgkEolEIpFIJJKrkiFDhvDEE09w4sQJFEUhLi6OuLg4Jk+e7JOvRYsWjB49ukzaWBZIoZFEcrkQUa2sWyCRSCQSiUQikUgkVyUffPABb7zxBlWrVuXs2bNs2rSprJt0WSDN0ySSskYIiGkEweXKuiUSiUQikUgkEolEckkQQpBlc/6rdYaaDQGHnY+IiCAsLAy9Xk9sbOwlbtl/Byk0kkjKHAGJeyB+F8Q2KevGSCQSiUQikUgkEkmpk2Vz0nT0kn+1zp2jbyTMYvxX67zSkEIjiaSsqdUNDBZw2sq6JRKJRCKRSCQSiURySQg1G9g5+sZ/vc6LQafTIYRvwCKHw3FRZf7XkEIjiaSscdnBaZWOsCUSiUQikUgkEskVi6Io/zmtn+joaM6ePev9n5GRwdGjR8uwRf8+0hG2RFLWHF3p/iGKzCaRSCQSiUQikUgkkn+P7t27M2vWLFatWsXOnTsZPHgwer2+rJv1ryI1jSSSMsctLIquX7bNkEgkEolEIpFIJBKJlxdffJGjR4/Sp08fIiIiePPNN686TSMpNJJIyhohoFJzsESUdUskEolEIpFIJBKJ5Kpl5MiRjBw50vs/PDycOXPm+OQZPHgwQgicTi0SXH6fR3FxcX4+kP7rSPM0iaSs0enh7HY4vrasWyKRSCQSiUQikUgkEokXKTSSSMqa+r0hsoZ0hC2RSCQSiUQikUgkkssKKTSSSMqarCRIO66ZqUkkEolEIpFIJBKJRHKZIIVGEklZc3yN9i01jSQSiUQikUgkEolEchkhhUYSSZkjIDQWql1T1g2RSCQSiUQikUgkEonEixQaSSRljRBgCgZF3o4SiUQikUgkEolEIrl8kLNUiaSsMYXCuSNwbFVZt0QikUgkEolEIpFIJBIvhrJugERy1eK0Q8pBqHcjRDcE6QdbIpFIJBKJRCKRSCSXEVLTSCIpK9KOw9SOkHIYkvZKR9gSiUQikUgkEolEIrmskEIjiaSsyDitfR9Y7E6QqkYSiUQikUgkEolEIrl8kEIjiaSs8Di+zk6CKm2gzg1l2x6JRCKRSCQSiUQikfgxZMgQ+vbtW9bNKBOk0EgiKStUp/Ztz4LUo2BLL9v2SCQSiUQikUgkEonkgnE4HGXdhFJHCo0kkjJDgarXQP2bICcFjq0p6wZJJBKJRCKRSCQSyVXLvHnzaNq0KUFBQZQvX54ePXrw7LPP8tVXX7FgwQIURUFRFP7++2+OHTuGTqfj+++/p0uXLlgsFmbPng3AtGnTaNiwIRaLhQYNGvDJJ5/41PP8889Tr149goODqVWrFq+++qqPwGn06NG0aNGCL7/8kurVqxMaGspjjz2Gy+Vi4sSJxMbGEhMTw7hx4y75OZHR0ySSsqJ2N81ELbYpVGqB9GkkkUgkEolEIpFIJGXD2bNnGThwIBMnTuT2228nMzOTVatWMWjQIE6cOEFGRgYzZswAICoqihMnTgDwwgsvMGnSJFq2bOkVHL322mtMmTKFli1bsnXrVh566CFCQkIYPHgwAGFhYcycOZPKlSuzc+dOHnroIcLCwnjuuee87Tl8+DB//PEHixYt4vDhw/Tr148jR45Qr149VqxYwdq1a3nggQfo0aMH7dq1u2TnRQqNJJKyYu+v8P190HsinN0mo6dJJBKJRCKRSCSSKxunTft40BvBGASOXHDlM+0ymLWPPRtUV750CxhMYMvynT8Zg0FvAGuGfxkBcvbsWZxOJ3fccQc1atQAoGnTpgAEBQVhs9mIjY0FQIi8Bf+RI0dyxx13eP+//vrrTJo0yZtWs2ZN9uzZw2effeYVGr3yyive/HFxcYwaNYo5c+b4CI1UVeXLL78kLCyMRo0a0a1bN/bv38/ChQvR6XTUr1+ft99+m+XLl0uhkURyRfL9fdr32e3at95Udm2RSCQSiUQikUgkkkvNqvdgxVt5/1veD7dNgYXPwtZZeeldXoBuL2pzpsPL8tJv+RBaD4Zp10PSvrz0+36EOj3gvUZgz/QtI0CaN2/O9ddfT9OmTenZsyc33ngj/fr1Iyoqqsj92rRp4/2dnZ3N4cOHefDBB3nooYe86U6nk4iICO//77//ng8//JDDhw+TlZWF0+kkPDzcp9y4uDjCwsK8/ytWrIher0en0/mkJSYmBnyMF4IUGkkkZY1QIe46qN+7rFsikUgkEolEIpFIJJeO656GDo/n/dcbte+b3oGe4/PSPRpCd3/jr2kEMOwvf00jgKf3+JcRIHq9nj///JO1a9eyZMkSPvroI15++WU2bNhQ5H4hISHe31lZWQB88cUXfto/er0egHXr1nHvvfcyZswYevbsSUREBHPmzGHSpEk++Y1Go89/RVEKTFPVS2uxIoVGEklZYwyCE+sh/TREVCnr1kgkEolEIpFIJBLJpaEwkzFjkPY5H1OIfxqAObTgdEt4wekBoigKnTp1olOnTrz22mvUqFGDn3/+GZPJhMvlKnb/ihUrUrlyZY4cOcK9995bYJ61a9dSo0YNXn75ZW/a8ePHL6rdl5J/JXraxx9/TFxcHBaLhXbt2rFx48Yi88+dO5cGDRpgsVho2rQpCxcu9NkuhOC1116jUqVKBAUF0aNHDw4ePOiTZ9y4cXTs2JHg4GAiIyNL+5AkktKhSmto0g9UB5xYV9atkUgkEolEIpFIJJKrkg0bNjB+/Hg2b97MiRMn+Omnn0hKSqJhw4bExcWxY8cO9u/fT3Jysk+ks/MZM2YMEyZM4MMPP+TAgQPs3LmTGTNm8N577wFQt25dTpw4wZw5czh8+DAffvghP//88791mCXmkguNvv/+e55++mlef/11tmzZQvPmzenZs2ehdndr165l4MCBPPjgg2zdupW+ffvSt29fdu3a5c0zceJEPvzwQz799FM2bNhASEgIPXv2xGq1evPY7XbuuusuHn300Ut9iBLJhRNWCaLioHoHEDJ6mkQikUgkEolEIpGUBeHh4axcuZKbbrqJevXq8corrzBp0iR69+7NQw89RP369WnTpg3R0dGsWbOm0HKGDRvGtGnTmDFjBk2bNqVLly7MnDmTmjVrAnDrrbfy1FNPMWLECFq0aMHatWt59dVX/63DLDGKEJd2ptquXTvatm3LlClTAM0DeLVq1XjiiSd44YUX/PLffffdZGdn89tvv3nT2rdvT4sWLfj0008RQlC5cmWeeeYZRo0aBUB6ejoVK1Zk5syZDBgwwKe8mTNnMnLkSNLS0krU7oyMDCIiIkhPT/dzSPVfQVVVEhMTiYmJ8XGWJblMGO12hNb1Rfh7Atz+OTS/u0ybJPuMpKTIPiMpKbLPXD6c+GUcpmPLiX1yaVk3pUhkn5GUFNlnJCVF9pnSxWq1cvToUWrWrInFYinr5lwShBA4nU4MBgOKopR1cwqlqGsRqMzjkvo0stvt/PPPP7z4Yp7Hcp1OR48ePVi3rmBTnHXr1vH000/7pPXs2ZP58+cDcPToUeLj4+nRo4d3e0REBO3atWPdunV+QqNAsdls2Gx5of8yMrRQfaqqXnLHUpcKVVURQvxn23+l43kdicR9KIBqDocyvlayz0hKiuwzkpIi+8zlw5FtK2jkOsgPG4/Rr031sm5Oocg+Iykpss9ISorsM6WL53x6PlcqnmO7nI/Rcw0KkmsE2t8vqdAoOTkZl8tFxYoVfdIrVqzIvn37CtwnPj6+wPzx8fHe7Z60wvJcCBMmTGDMmDF+6UlJST5mb/8lVFUlPT0dIcRVKTEP2vcjhpT9ZHZ6qaybUiCx7m+bNQel2rWkRraESxwusTiu9j4jKTmyz0hKiuwzlw8JVKCrsgHz5s9JrP5kWTenUGSfkZQU2WckJUX2mdLF4XCgqipOpxOn01nWzbkkCCG8jrEvZ00jp9OJqqqkpKT4RV7LzMwMqAwZPc3Niy++6KPhlJGRQbVq1YiOjv5Pm6cpikJ0dPRV+fBTVm+B+B0ExcSUdVMKRCg6FKFijqqE8s9MYpRUiK5fpm262vuMpOTIPiMpKbLPXD404CgAMcZcYi7TdyXIPiMpObLPlJDDy7XQ5XWuL+uWlBmyz5QuVquVzMxMDAYDBsOVLXI4XxBzuWEwGNDpdJQvX97PPC1Q08FLegUrVKiAXq8nISHBJz0hIYHY2NgC94mNjS0yv+c7ISGBSpUq+eRp0aLFBbfVbDZjNvuH/tPpdP/pB4eiKP/5Y7hgytUEUwjK5XrsQoXWQ1Fa3Q//zER35h+o2LCsW3V19xnJBSH7jKSkyD5T+uxa+Ck1O/UnJKJcwPt8wD18yStgDrvsr4XsM5KSIvtMCTi4GPRGqHdDWbekTJF9pvTQ6XQoiuL9XIkIIbzHdjkfo+caFNS3A+3rl/SOMJlMtG7dmr/++subpqoqf/31Fx06dChwnw4dOvjkB/jzzz+9+WvWrElsbKxPnoyMDDZs2FBomZKrlEN/wZavyroVhWMMAUUH5nCo1U0TIkkkEolEEii2LFBVzh7cSpONz3Nwy7IS7V41SlthVLh8fTFIJJJ/gYzTkHK4rFshkUguUy65GPXpp5/miy++4KuvvmLv3r08+uijZGdnM3ToUAAGDRrk4yj7f//7H4sWLWLSpEns27eP0aNHs3nzZkaMGAFokrKRI0cyduxYfvnlF3bu3MmgQYOoXLkyffv29ZZz4sQJtm3bxokTJ3C5XGzbto1t27aRlZV1qQ9ZcrnQdlhZt6BohAu2zoINn8GR5XAZO1CTSCQSSdlz5th+csZUIu2sZlaWPrEpxybfyLnT2mQvvVyzEpX3yLl3eVH/DP9UvBOA07+N58ymBaXbaIlEEjh/vQmpx//9evf9Bgf++PfrLYAMq4O4F37H6nCVdVMkpYB0LF72lMY1uOQGhnfffTdJSUm89tprxMfH06JFCxYtWuR1ZH3ixAkftaiOHTvy7bff8sorr/DSSy9Rt25d5s+fT5MmTbx5nnvuObKzs3n44YdJS0vj2muvZdGiRT42ea+99hpffZWnZdKyZUsAli9fTteuXS/xUUsuC6q20bR5LldUl6YKnOYeHERULdv2SCQSiaRsyE0Fpw3CCjbd93DQXo5WqiAl6QyRlWoS4TqHNV3ljNXBj67rsBAacJVOWy4V1QSU2AY4bFrAjyqb32a/qQm0ve2iDkcikVwg27+DujdAVI1LV8fil6HzKAiKykszhYI938J6dgqElL90bSiCc1l2ALJsTixGfZm0QXLxmEwmdDodZ86cITo6GpPJdFmbcF0IQgicTicGg+GyPDYhBHa7naSkJHQ6HSaT6YLL+le8Uo0YMcKrKXQ+f//9t1/aXXfdxV133VVoeYqi8MYbb/DGG28UmmfmzJnMnDmzpE2VXEkseQUc2WXdisIZthR+HKZpGNXrdVU7H5RIJJKrmUNfDKbOuRUwOh2AMZ/PoUGL9tx9TZxPPmP6McKUXM46Hez7+3sqilAecT3HzSHtqM8ULEd/huaPF1tfjt3J/gOHaakIblGX0WbLXHbYn6YZ8HXcBMZdgmOUSCTFYM/WzMSyky9dHaoL1k2BFvf4Co36fQlLR2u/XQ6Y3BSe3u2b519CdWveO11SA79MWPcxWNOh28VFn9bpdNSsWZOzZ89y5syZUmrc5YUnjL3Hf9PlSnBwMNWrV78oX11XtitzydWNMRga9NGEMpfbjSwExDSEZv0h4wz8MwNOboRq15R1yyQSiUTyL7PfWZE67t+qy8XLpx9lnPgYzhMaBaXs0fI4bTT4+2FOUYFKhiyCE/6hpe4Qux25AdU387s51D30JVuUumAKw4iTZrsm8perJeUNtos6lsOr5xJSoRqxDdpfVDkSyVWHVRMao7uE0zOXpsXDkRVQsXFeesYZzUcaaP42HdngyC0ToZHdpZnSSPO0MmLtR5B59qKFRqBpG1WvXh2n0+kNTX8l4QljX758+cvWebpery8VTSgpNJJc2VRtU9YtKBihwtgYeHqfZpawdRac3Z4nNEo+CLlpUK1tmTZTIpFIJJeeDRUHUCttPQ1UF+nnEohSVG7KmQ/c45NPddlJEJGkRzRhhrMnQw2LeUb5huTEDoQqVkSAQiMlJ5lWugN0133Jp6bV3vR6yinC4r8CLjyCUuifo1iidOL+0VJoJJGUCI/QaPOXkJ0Ere4v/TpcDu07N9U3ff8fcO3/tN/pp9x57aVffwBYHZrQyOaUvnDKAmHPpjSX2hVFwWg0XvZh6S8EVVUxGo1YLJbLVmhUWlzZRye5urGmaaq2578YLwc8kdJWvKW9lOve6OsI+8ueML1H2bRNIpFIJP8q1ZwnGO+8B6sT0pNOA6BX/SdswmknWUTgEIL2ur0AGBUXikvzSRSo0KhN7hrKK5ncqV+JYghisasN81ydiVHS0OWbKJ46uL3Ex7JbjWOfo2KJ95NIrnqs6RBeFY6ugF9GlL5D7C97wxy3INpp9d2WnaQJjjztAHA5S7f+AMm1axopUtOobMh0XJzI6Gx6Lk98t7WUWiO5XJBCI8kVy7YWms8rcf6L8UJIP1W60c1U94vw4FJYMxn2L4T8IY+bD4RrhpdefRKJRCK5bGmUtZ5pxknkZKWRkZYCgMmR7pcvy1SRxrrjhJ1eTZjezuqqD2EUdnROKzOcPfkn+s6A6lMNQQAMVn/meO172KnWpJ9+JVlKMDq3sOrUoV1Und0Za27JfAN2129jnPHLEu0jkUiAmEaayZhn3PrXmNIt/8RaOLYKur4Ih5bCn6/nbUs7rqVBnkPsoCjY9ROka4JskvbD6AjIOFu67ToPq1MbI0tNo7LhM9dtWMWFawX9uv0Mv26/Mn0YXc1IoZHkgli6J4EnL3MpcrLQoshkZudcfGHvN4YDiy++HA/CLTTS6fMcHsY0ytueegwS95RefRKJpEwQpSlsllyx6BUVs+LAlpbA6bDmDLc/hcWZ4ZfvdLm27FBrorocWIQVEVGNfdTipKkmu0UcNmdgK/Mud2RRqzARRhajjHMByFDCsKJFV3HYNGGR3VayhZf9qowEKpFcEKYQUB15/xveUrrld38VanXLG1/q8kUmi6ye99uWBdENtehp84bC9Bu19AxNeGTLTCbuhd9Jz8nX1lLEWoSm0e87zrJg2+lLUq9EY6n5evrbX7vg/dMuUb+QlC1SaCS5ID5fdYRfLnMpcssVwwDIzMoqJmeApJ8snXJAc9I96hAERWoaTA1vgZrX5W2v3ELzaySRSP7T7BvXgW1vXbh/GMnVgeI2WbbnZkLacTII5knzm375yiVtopnuKMJpxyJs2GOa8z/xDAvD7qKeIYFrj0/x2yc3K52MlATf+hCkiRCSnUFUTFrnTX/X8iSflxsFgBpcgR9d12HVh5ToWN5x3s1u9RKGC5dIrlQ2fwmHl0Gl5tr/Exsurry0k3Asz2cZUXFQszPsWaAJjkJi8rbd9ZXmABs0f6DWdEhwC5eqtNK+a3eH4PLY3ILk+Ax/gbIQ4qLNyjyaRgWV8/7SA/xvzjZpunYJeUrMor1uD6nZF+bTKi1XExrJRbMrCyk0klyxqE4rrzqGcFSNZe7mixT4tB4C0Q1KpV2A5tPIngltHoQKdWHvr3BgSd72mMYQUqH06rvEuFRBlq0A23chwFk2jhQlkjLFrmk4Jtv0/JpZr4wbI7ncORVUHwB7Tiblzq7kf+bfMOck+OUrl7YLANVpZ67ahdCQEAa5fqZ92m/UMaZgsPmbtJ384EbCP/Ltg79Xepx3nf1JqNCO/B5PFZ2BWtmaHyO7JZpvnd2x55Rs4WWQfgkNlBMl2kcikaD54gSo11v7zoq/uPK+GwAzb877v/dXSDmU93/Dp3m/Dy+DsMqa+4TgcpoWkjUN+rwPvSdqeeJ3wS0fkluuodbcAgQ301cfpcGriy6q2bYiHGFHBGlmU6sOJl9UHZLCqas7zUvG79h49MLOsccnlVOVQqMrCSk0klwQussggr3dqTLoy43M2Vjw4FTntOFEz4zVh3h23g7fjZnxcCZA8zqXE5rcCdVLMRJMbhp82BKa9YdO/4PIGpC8P2/73MGQsKv06rvETJ23kOlvDvNLP7LoIxgbXQYtkkjKjrSEEzC+Eid2rKC+7hQGgwxUKnFjz9Y+57E5sjffObuRgwVhzyZSb+NH9SmE6vJZrRUuOwtd13AwphdjHIMIDwtllP47OmUuJkaXibkAP0iHbJFsUn2FRqFZx4nr2J87/vc+Ln2QN32g40f6p2v+iETqCX40j8FVQme8tXRnedN5CaI+SSRXOtZ0KFcb1k/VvvNrAl0IxiDf/3t/gW2zIbg8DFvm+yxa9zEoCqyYCFtna9r1Lge0HgrvNYDPumhCp42f4XBrGmUXsFi454y/We32k2nYS+CfyOHS8h5M9BdYZ1odGPUKZ9MDc/ovKTkOYzgA9owLExp5hEXSJ9WVhRQaXaUIIfh63TG2nUy7oP2VUg3GeGGsO5LCygNJvPDTTr9tjV9bRHZ2FhOM06ll04QxPmqSM/vA510Dq8ieCV/dAof+KoVWu/H4NPrjec1GvEqrvIhqAHpz6dX1L5B5aC3/M/yMUH1XnVL2ry2jFkkkZUdilqZdl3hsDzFKGoN0f5RxiyRlyd4NeavuKW8348w7/gsQzVP+YLHahqSQugh7DlnmiugUQWZqEuMnjuWTZe5FBZeTckomrtQTfGCcQniwBYAgVyY5lhhMTn+BVIoI5x9V02Ta/9tkDi37ihviv6DxOc1PX25wFXapcexXq5KtC0MvtP6rZGom6A5ryfwCuoSOU0JbLPjneCrz/jlVov0lkqsWa7q2SBlVAwwW/whnJeWGN+DeH/P+e8aW3V8Bc2ie0OiXJyHloCYoOvwXuGxautMG09yRfGMagi0Djq5EOb0JgPRcf981Rr3/1PK2j9fw9bpjATfb7tLG6x/+dZAb3lvhsy3T6iQ61CzN0y4hLrd4QE27MI1Rl+o2t5ZCoysKKTS6SsnIdfLagt30/XhNWTflgsm1Fx4KNNvuoqdtAifUaEw4+Mz4Hva/JuRlsPmvhBSK0/3yPLryAltaAB7hSvxOWD4Bdv/sG52tfG3o/3Xp1XeJuU38DUDmuSSf9LXhvTkl/jtmdhJJaZCuL8dfrpZkZabximMoqfq8e8Bpzebop3f7CVglVybJZ47T8I+7STx1BIA19rrMyunol6921mbeNMzEmLidDCWcc+ENcQodp/Zu4IWcSWw7oglwzoQ0pL1uL3VOzOUW3TrCIiIBCFUzORbbiwdNb/mVbcTBQL226FF/8+vUWfkkRmc2OnOYltasPf+L+ID6ulOEKlaCVW0i6XJoE1anvWQr+jV0iUwzTQLg1yVLWPSjjKQmkQTEjWO17/gdULub5kPoQvmwJWyaDhn5hLb13A6tm/aHoHJQt4c29tzyVV6eU5s0DXudUXOEfXozVO+gacRbtbGzw24DBBnWAoRGBm1R+XyBgasAU6U2Y5cyf6u/U2uHS6VLPU3wfL62UUaug+gwM1aHFEiUOg4rqC5mRz/FLjWO8mcvbN7jcHk0jeQ450pCCo2uUhzqxT1slbJXNCI+vegVmCpKMk70GIWNnvrNvkKfkkzYPCs9rlL0zSNU7YWs6MCRA4YgqNEpb3vuOdjxQ+nVd4lRdZr5TXZWmk/6CWMtetjeKYMWSSRlh3J6MzaMrHA2I5IszOQ9O04c2knN+EWkJl3egQQkpYPLHQnpTKb23VA5QTnFf9FCESrVdElEbp3K6rBebKk+lGSiyIo/jF4RPJk4GoAD4R1Y6LoGnS2NXEwEBYfxs6sTv7naYS5fnTjrPr+yV7uacEhU8f7PEhZCHCkYI2MBiI2w8NczXUkVoajNBnLWGY7VasXl0BZMPKv+gbLS1dT7+/ZzXzLNNAlxkWMOieSKZ/1U+H1U3qJm3Rugcd8LL+/cEdg1D379X17aXW7h0IQqEBqtLU7mpvrvqzqgfm+tTQC3fAirJkG5OAAqbPmQY5Z7C3w2GHTa1DItV3vveUzNgs3+ZtrJWTb+2HXWL93pUikfYqJCqAmjXsHmdGFzunCpgmy7yy00kgKJUmdcRewfdyJbmLnbNZYlFQZdUDGeayM1ja4spNDoKuVib2SP0KgsPeNnux2tVY0KKnD7b6aXKa9kYLKdA+CcNV9b84cZLQ7PMW76As4dvaC2+hFRBV5L1k6kULWXc7W2edvvmAb7fiuduv4FdMLJQbUK2cLik943cSprzE/KCYPkqkI5d5gKSjr6cwcZZZyLTc173mQSDEBaojTZuRpwOlwkikiO20IBzcHo3fq//fIpbpPlFunL6Lz9OeJs+7nD8gUnrdr7zezStH8aJfzKTfqNGOwZWBULBr2eZ5yP87pjMBUiw5mmvu5X9gvG72it06Jxxlm/pYntS7JcBoLKVfXJFzXmNJ163c394g1OpjvItFTmDcf9JEe1KNExv+O8mySh+cTQq5rgyeGQIZglkiLZ/wfs/AGOr4Vq7TTflysmXnh5Na71T9s5D2LzhLpsnQ1/jdF+d3xC+45tBi3uhaxEqNUN+n0JlghNkNTxf1CtPcYsTTvIUcBcwuPHJjVbu+c9JmzBxoLH3c7zBE9Ol8q7Sw5gd6l8OKAllSODGPj5em79aA1ZVs3CoEKo1DS6VJhS9vLayWE8Zv6dKqkbwVFy31EeX1fSp9GVhRQaXaVctNDI7dOoLB8IDpdKg9iwAh3xgcCiOOhle5t/hBb1zGbPpyk0+Fd49khgFZWrmbc6cy7AfYrDkQvH1kDn5zR/Rrt/gm3f5m03mLTv/0i4ykRdDOOc95Kui/BJD3cmU17JxG7LJis9hfg36ksBkuSKx5WbwTW6/TyS/j7b1NoMN+WZxp4lmttsb3BaiS3DFkr+LXJDKjPX1Zmo05pfjtccg/mHhn75tod05ISqmWPcoN9CXPY2rjEcxJ56igNqFSxuk7GKWVoI7Hglhi8NA1AUhbGG6XxveoOKUaGYFYfbdCSPqkqeM9PbdKsZol/EQOsLRNRp59cORVEYZPwLW8pxUoNrckBURSmhX4u3jV8QrWSw7cQ5cnI1f0g2W8n8IkkkVx1RNbTv8MpgDIaz22H5OE0z3p4DK0qotX18tX/a+k+g/s3Q7lHtv8sG/7jHt80GaN+1u4HeqGnZ56RAg1vApC12sPp96PsJp+rex3xXR68W0djf9nA8RXtGecbk59zh2tNy3D7SCrBQiFPOorjynlcpWTaS3T4Bz6ZbiQw2kZptZ8uJNPYnZJJhdaBTICrERHxGLnVeWig1jkoBIQRvL8rTUi3vSuIaZS/Djz0Fs+4ocXk5dqlpdCUihUZXKaUl7LlUQqMf/znFwYTMIvM4XYIwiwGbU2V/vG9eM9rKRg1dAiHWsxxRY1laIV80l4hqsP93LTJEcWQlguoWTFkiS3IYhZNxBmbdrtmXd30JanaG/APzmX2070DadxnwXugz3KD7B53bOaIHndDOW25mBidPn6aCKxG5OCT5L7Hl8FlW7/P3uVAUaq4WwaoCaegUhdaOf7zbzEeX0ka3n3MOGVHtasCZncZw/W+YszXNspv1G4hVzvnl2xjSnUnOu7z/FVMIk7OfZ2DKFD4I/R8Woa326pxW3nfcySemwfwZooXSvkf/F+10+4iOqQRATmaaT9lH1Yq86bgXgA9MnzDa+DUzwqZSObJgLd17xe+QfJCI+LV8Y5pAeNI/BeYrjPrKCW62jeP47Ce5Rqc58PZEW5JIJIWwb6H23flZOLIczmzR/qsuzb3C8rGBjwlTDkOTftrvmp21iMGqS9MWqtwCert9n4XEAAK6vgixTSC6AcRdB2s+hLPb4M9XYf9CMIZo+f8eD/sX4nI66Ktfi8PpIu6F35m2+iizN5zgmnFL2XVae/95hEUHEjSfRB4BU37+Nj/DnanTvP9bj13KHZ9ovlZ1ClQIM5FhzVsYzrQ6CTUbCDLqOZKUjVMVbDrm/zyVlAybU2Xq34d90nL1mrYoaSWLngl5QiPp0+jKQgqNrlKW7I73/r6QmzrXLdl3FvASKA2embudl372j4qWH4eqYjHqybG76DnZ11mbGQcZIoinDPOoYD9Dd/t77DS3zMvweVf45QmvU78iObsDfnwQeo6Hqq0v4GgKQHVp/ozmPQgn10N4FV8/SzqDttIk/hsSlna5q+ih/wfSfU1uPKYJudlpKI5sDIqKQ644S/5DnJk7it3fPFuifQ4EtWSpS3veBOmcjHF96N0WnH6YV42zqXTs51Jtp+TyRJd6GIOiIpzaBKqdbh8G/LVje6V8jU3Ji5qpN4d4f1cP03Gz+p6W7sohVjnHjWk/MNL2mTdPiggjLCyS06KCV7sHpx01KxkdglPCN3S3Ely+0DZbdcE4cjIwZ50EQC3B4oVQVfSKIErJoo44zlj9Y8RZv8VqjAy4DInkqmTQArjuGajaBszh0P01LT31WJ6mT25aYGUtHKX5MwIY+D1Mqg/rPtaETjpjXr6gKIisDl1f0P4n7YPZ/WBFPof6EVVBp4PydbX/K9/FgbsMe56TapcqSMy0cSRZ0zg65xYaPTZbE36d7//I497CkR7PlGUHveln3P5KdYpChRAzel2eitKaQ8kEmfRYjDoSMrR8GbmFB8WRBIbHSfl1tve9ad5ndoObS1xejl2ap12JSKHRVcqkPw94f59vTxwIme6ICY4L2DdQCgrbmR+HUxBUiI10BiE0s00nUwmjousMzxrmMOrIA3kZ3I6bA4qi5g47zOKXSs88Taia0CjjNCwfD9u/8xUQGYO1AYTRUngZlxH35s6mopKGMzfvfJ7dtwGzNYUlrtakZ1nRObWVcmdu0RpkEsnlRIyayHDD7yXaZ5+xAatNnVjrasSUyGfR5xMSCLdDfeGwFba75ApCdeQLXQ04hJ6RyvN++Wra9vCgYREAM5034oyIY0HPtfzpak0rwxHCHUm4XC7WBHWjkiGTJ11fE6tL8+6fZKiMTq/jRvExmUYtWt+2uWPRvVub2a7r+cz0vk99rqDChUZ2fTAuawbCHTVNOAMXGjndwrFvTBMIdqbTp2NzOhv2YM+Rz32JpEgq1IPur8L277WxaflamlDHadXGiwDv1im6jIm1tYVOD8Zg+NMtfDKHadHYwvKZRkc30Pwnbf/evyyPT6Twytp3iluwE1aJasd+1H7n83dT89xqrlH2okPFbNB5/Q95ON9UyeESvOG4n+nO3ry75ADno1MUdDrFJ+rauIV7SciwYTHqSc3xzEOkYOJi8cwDg9HeUyfUaHZFdCVRFwM3uc0ip/eEzTMCKi/b5kKnSPO0Kw0pNJIEJDTKtbt8fAf9Gw/rooRGJ8/lsPpQEkGmgoVGlUihlXIA9CYaK8cYpl9IRfvJvAzuiDbYAhjIJuaLRnNifSBNLx69EaLiALcj7PJ1fKNkuGyw4m2wZRVSwOWFXrhIFyGo1rzzeWznGmrrzvKc42FSVn6G9ZhmuuaQk2XJf4T0M4dpaN9d4v26nfiIyGAT01w30TAkG4PIp0XoFh4Ip7wP/gskZljJtV+4ir3T/bzbeTyR0yePYlRcdGC7Xz5FqMQpmgbwX2or7JXa0LqyiRv0/xAWFctS83Mc3LSYfgkfsN/SDABrsGaONt/VkT8i7wbgBsseMs8eAuCgrhYAK9XmqELB5XSyjuYAqCEVC23zntB2nDPEIKyamUm2sVzAx+twCWY4ewIQqmZiDqvAx/pJiLTjrF/8HRlpySD92kkk/nzRXXOGXbs79JmsCYwMQZrQKK4Ap9YA6ad9IwPnJMPJDXB4mfbfkQNbZ2nCoxqdoNcEzQzNQ0h5zUTt7Dbt/41jfcsPqwyh5z0rwisTnq0FhXHZ8zTH+x17jR/Mb/Ka4Wuiw8xeE6WoYCPVygV55wvWjV9xavk0cm0OZruuZ7soWBDmcmsi3dOuuk/69w+3x2LIG/tLoVEJOPI3/P22X7InovZM00TecfSnipJMcmRTJoU+DafzmSebQoutwqUKch0uyoWYpKbRFYYUGl2FTFvlqy3jLGAAN2vdMdqM/dP7//ZP1nDdxOWAplKamu0bSvNCybW7vJEVzseoL8Brnpu+H6/hQEJWgZpGQgi66bcx0vAj6cZonBhIIxQTDu+EDZcDQmM14c3vz/iGJD2f0Ji8F7g9uySHVzjla8Nja93R0wRUaeMb0eKZ/XBoaWBCrQvgdFouR5JKTyClx8lc4y2c1cWSdXgDAIp7YDI4dBOdkr6H7d/hEgpOu5wsS/4bGD7vRBjaPe+wB+6TpVLOASpaBO8YP6OJ8RQHRV6UqkPh15AqQvOeRZLLmmvG/8XTP2y74P3tipmFrmt4yDWH0z88Q7Yw85iYw9avRrH9p3e9+XTCxSmdtqI/y/QWkfazlIuuRIYIoua1mq+jowd2UUk5RyNF8zGhRmiOc0c6RuCsp5kQDNX/gWOPFnnzWLgWkXOR+QV0iiAjNZHXwt+kpvUbMpsOLrTN6yvey76gVuwPacNA+8scq9Al4ON1CANTnbcCcI/zNUxVm2FXTDhtubRf9wjhk2uzd0q/gMuTSK4K1n0MCTshuj6ERkOboVq6JUITAiXuzcubP0DKmS2w5JW8/xWbau4O8uO0Qb2emm+aZWN9/Wc6rLD+Y3BrwFI+nwBn0Hx4cmtetOGBc6BiE6igmamtcjXhtDXPL5pLaFPKKkoyMWFmrxsLq0OlfIjZG2nNsvBJqq54BmvCAfZbhvCKYVaBp0R1axi91qcRk+5q7k1vXi0SszFv+nopLR6uOH55QvNLdR4ebS4jTlapTVGAOsE2ou2n4a83tEwn12uO0YvBc92Ts+wMn1Uyf3iSyxspNLoK+XK1b9h4p+r/wF28O8EbwQBgX3wm57LtnE7L5f0/D3j3KWjfkjB05kaaj1lS4LaiNI1S3EIrSwFCI7tLJRgrGYTwfdQjfOfqTgZh2kaPD6MH/4SROyCmIez6Ef6ZWXBFQmgvyRb3aKs/9lIStOSmwq6foOc4zUnhjjmwMl9kjHi3erGaT6CWXfzDOlD6fLiK7pNWlFp525RGbA7pTN99ozQH38C5yKbcFr2Q1uW1ybEBzdeFPZ/fDonkciaEPNX7nOzABbgWVyZxylnKK5noG/TiNvubXv8NRyxN6WabxLaKd5Z6eyWlzzHLPdx3/JXiMxZCSlQLdqlxADhUHffYX8aFnpZHv6D5jje9+RaF3MZac542QbjeQXBwKOFj4qkYW5kcYSbUqmkiqToDh9TKRLXtD8DWV2/g6RvqA2Ct2YPqB79BqCr1T81lpvNGb5kZKfH0sC/nqbYh3NCsRqFt7qDspMmaJ9iWoicEK+XO+WtGFYYjM4k5pjexCSNZTj1hoWE4MOLMJ3StkLKpiBIkkquQxS9pWj3lavmm3/+zJhTa8wtEu6MuuvLG5mz5Wouy5iFhp6a93u/LfIUIzRF22gnYNB1yU1FVQUqWjcOp7jGmx29ZhXrQyi1QPnfU10WCOQxSDrOj6Yvs19dltqsHp3LzAjqECG2BZb3akJgwC9k2J6pb6yQy2OhdZE7SRXNKVMBxXFtgrKIkU1vxDzbhmV9YjHrubF0Vk16HUa9gNugwS02jC6OQSJgOp4sXDN8RjI1ySiY6RVClUiwZLqOPCaJXg60IcgqMaC25EpBCo6uQyGCT97eiFPzANbi1fDz2qAa3I7pOby3jw2Wa6rvFqLtoe9W9Z/0nYp7JVXE+jQCC85mneSTlNqeKGQc2DDTRHSVWOccmQyseNY3PC2Xvsml+hM7ugCIcgmLPgtl3aiq+sc0gvGrheUtC2kn4/Wmo3BJueANaD/EVCn11m7udDk2VX1XhnVqQfLDA4kpKVik/1F/TPUFHNMfl+1VttTw4ZRc32RZjrNqK06I8Hzn7clCtgtVcxPkuJe6dtt4b/rWsiN+xjLRTe4vPKLlsucE2kafsj/KtsxvWEgQyDHZlolZqToKIJK52A+7T/4nLqd1z7c58xQD9cpz/kciIEogWycVnKgRD2hEG6JezyNWWb0IH86jhV8zYmeq8BafIe8dtNLVHqXM9h9TKrK/zFDXqt/Qpx4iTCskbAdiuNKSH/V0a1tMERVEhJq+z2Oa3jKCCmsQ7cxajO7EWk9uf1vX6L0k01+Be23d0r5jt41z2fGoY0+murmNywlCmmSZR8dzGgI/XZcsiTkngFvtY/jKPIipIzwl9NeyqwixnD3aoNXkvX5Q4iUQCjE6HZ/b6x6U/tkr7Vp2ag+xXEkGfN4b3OrX2aB/pjFCxEcQ01v63eRCqtYcT6zRhk9MGBgvTVx+l9dilXP+e27QtppH2Xb423PKB9nvFeWZMcdfifOYAw6f8wll7MG8YZxKVtosKoWY+v68VAAkikumum4kKMTJ7wwnWHdHGtRFBRmzuuca9YdO5zjYZ0xHNmuF63Rb+ND0HaPMKD6rwXZQOMukJMRtQFMUnnxQaXTwuey6PGH4lWLHxfCctalpsdHmyXEZt/gNa34oofg6UY3dhCmD+JvnvIa/qVUhEkPaSGdmjLobznMx58KR5hAsF+Q4KNRsuWtPIZPDvgtluO+hAhEb5NY08Lw67UyWVMI6rsXSxryBOieeb8IfY6KwLH7fXwt1/1llT0z39T56DwYKwpgMKmCPg2pHQrLQGu0Kr95s7tZUipw3y+z1RnVCrq/aifyNK+w9aRI1SoCANrYthiGsuVZUEAGyY2PNeH7r+3Y+OtpXE1G9PFSWFakoidXWnEakFr3SUJkOPv8D6TRsueT1FEfvT7Zz7alCZtkFycWQLC48YfmWV2owsxTc8+a7T6d4IIflxJh5ksnIvUU1vouKY41iEnbHGGTgdmqZFVO4JXjDOoc2JwBxKSsoeVTEUn6kQwpK3UV2XRG3lDBWsJ+il38QOtTYfOW/nVvtYju3TIgs9lvYOrcIyqPPGXtrfNxpF5/tees95F0HWRBa4OjLggafY+PL16AoQ/FiCQvhGdytd9o2mj34D9xi0leG4MJXM1AQMwoneWLS2Z2hsbd8EV+CLDA67DRc6HjMs4LghDqPRxKuhb5AU0YylamtysKDKoafkambPL5pZWH6OrNA00M9nwQjte/8fmqZP4p48rSDI0wSyZWjjVdUBp/6BT9rDfT/CTe9C/V5aHqdN849ksPiEqbdF1YUKdfh56ym+23giT3BVQKCYA6kQp4unlnKGbGHGkZPOzcbNNDv6Be84+lNRSeNW3RqvH7h7p2njsMggIw6nNl9olb2aj4xTiDmpOf43KS7+VpuzeHc8VkeeACj4vHlHsElPqFl7FvuO/aV5WqCkEsGfLv8o0A6hXfOB9peJ7fIwPLCEYLORBGdInrlj6yFgLt6nUbbdSbBZjx6tDxQ0x5T8N5Fv7qsQVQjG396UkT3qYdDp/B64f+9PZNVBbWXV6rZNPf/hDWDS6y5awl+QNPpMmqYKGWQquHuKfKsP+X0a2fMJjb51Xc+HrjsIDg6hrW4/fdWlZDtckJ2kvTRVJ1giNU2i3hOBQlZdremaoEan03wMHVp6YQfqdxCqVqfTBus/0bSe1POERn0mQ0i0tmLkEShlJ5VK9aUtNBqqzifIXeS74j4aZWirY6rOSI1a9TktyvOIQfOzQVZ8qdZdED30W6ka/9clr6coZjuv56ClSfEZJZeEz5ds4eAp374mVJWNX47i+J7ANCfWWp6kvu4Urxu/xpWw32fb+qnDCR7vrzVn+KQNPRwrCImKBkBv1FaFHXbNpECvevyq2f32lVyeuJSLeF66J4d1dad5IPMzUkUo99lf5BvTeKYaJxM3pxsAlZ0nMamFa0dGRwSzS9RkbfO3iSkXSUxY4ZE1vzDcSzvdPraqdckWZt5yDOBJ62eEHVmIgeKFRuWq1vX+zhaWvEWLAHC5HX/31a9FuLUgOro2oqQcZLrxHdrr9jJEvyjg8iSSK44f7tecU+fnp4cgpYDovB5/Qs5czZ3C510hOzFfBgWuf11bAJ3qNm89uAQQUL2D9r10tKYlEnctdHgMgiK9Lh4A9rV/F6q05qnvt/PiT5rGOI3vcO/vy6crDnNSRFNdl0S0ko7dlktNfTKx6dtpGZ7JRrU+d+lXeB0glwvR3n/BZoN3vvCWcyJ99FpQmed1zxAvouiu38aGw4k+dU25p5XP/6AChEah+cqVFM8o/XO85HjAL92pGBlhf4JYzhFmdEH1dliMetY76yIGztH8q276AowhxdaRY3dxnX4Phy33A3k+jkqDk+dyis8kuWRIodFViM2perWNDDrFxxG2EIIhMzb55AUoH+I/yDQaLl5oZC5A08jzUPCsSpxPTr5INvlVVD1R4GxOlfa6PbRSDhASGk4T3TE62VdjdagInUFbNXU5NcfWtiyo1ByeP1ZwA4WASlqkGo78DQcK9r9UYkyhWphTjyPsuOugw+N520OiYenrcHwtJO7OUz32qCpfJPnPW2lgwIUaqoVxraFLYq9ajXQRjKoYMRj0HK9xF6fRJtGqo/jJcvrhjTizUzWTntERnDpc8ghWekfZRZ5TXSp1dKcJInDnyZLSpd6qkWz7/QuftFy7gybHv+bExl+K3X/9l896f8cqqeSe8e2Dwwx/AL5CbA836v8hQq9Nno3uCfrMVZppqc5lI1uYUVQ7Z9NzefjrzSU4qpKzeO1mvvnr0tZxpWK1O3nBMYzvo4ZfcBnmo0tZ7tKcuFpEDk70PKZfQCvdIWro8iZJOuGCIjSaGlWvSB/9erqkzy+2zmfVaQBUH7GAeFGOEyIGuykKNSuZtTRDF1J0NLSYSnHsUWtwSlRgg74lCUG1isyfnxxLDM8rI7U/bo2FAdYfCEvYgEHRxgsNdScL2VsSKNtWzOeVl0eyP/7igmUIl4Ps+EOl1CpJQIRXgaiavmlOKxgKEOY2cfu+azUYWg3ShD/5gyjc+KamhTTrdkh3a3FnJ2rR0kwheZr0tbtr5m03jiWDYO/iLMAxc10IivIKZLT22ICCx+BnhbZYkkMQFhz0zf0RjvxNj9w/OFD9bkIUK6/2aeTNP/Ca6hjdi8wrDvgufFYK0ZznAxw85eu3s0Ko7/kIMRm8bfTMHWLCzFJoVALSHTreNM70SxfZ57hNv5b3TVMx2DUNM4tRj1nNxbVpBnii5F37VLF1ZNuchBtc7Fc1U7aCfBxllsTe382BhEyum7hcai6VIVJodBVic6reB65Br3iFLZBnGpaXV/sfHuQ/mDXofPe9EAoyT/OsgNgLeRHkl1ob8mkqOVwqqiqwO1Vu1a2lm34b5op1SBWhGNyr/egNmqNARzbUuR4qNoafHobFLxfcwNgmMMStIaM3ab6QCuPU5sLLOZ8KdWHgt4CiaR1FxWm25B6e2QsnN8LxNdp/UzA06EOhGlElJH+40tLAgJPsSu0AeE35gt72t3nE8RS7w7SVKsUUzHE1msNqJVzOPKHR3g2LWTHhNr/yImbdwP7P7ic5XosSdHjbSr88hSHcQlClqGt1ibHbc2mn28fi0DvKrA1XM6mp5+iq306s3le9PssuWKq2Qn++34gCaHN8mvf3caUKtuRj3v852Xnl2uza4CctJYltq371hhsPC4sEwGAykypCqbxxLGnZNv4I68cKXTtwOTiwYwP3HRxJbk7prJ6lxR/n6BZfbUh16RucWfZZqZR/tZGUaWWOqxtnghtecBlrbXEsK3c3AGkilCXGbjxn/N67fZqzN9lWB3pc6PSFC410Js08MspVvH+lji4tYo3JHESWuSL3Vj6L01IOY9YZRjkew1C+aCGQzmAgZ+B8FrraMT3iCXZFdi+2Tg82QzirjR3JEMFMD30UAJdiRLWVnRD/SsS46m3GGmdwevH7qCWI7Hg+2+e/R8in/uYqkktIVqKP6ZfDpSLcvob8cEcqY/N0iN+p5cmvpWoKhVP5NGdjGsGZrdqiKGiCW70JwitrwVbm3MutHyznbHpen8mwapN6j1YQAD1eh5b+5vU5dicu9MRZv+VZ00tsVBsQ6TqnaSPqzWCOoJXuEJW/7c6o2qdIzbFj0is0SvmT5sm/8/jXa33KayH2Ea2kAbD7RNFa6EEmPaEWX02jmHBzoXMFyXkk7mOe7kV66f0DEajWDG7QuyOduX1mWYw6zNgxLHwKJtXTtp1Y67fv+eTYXVTRn6OO27n5+fPKY8nZdHu35IF4krO0Mb0UEpYdUmh0lTFyzlb2ns3whqvU63Q+fonOZflqgdgceSZf52MsgXnazDVHmX5e1DYoWGjk0W4qzMm2LV96fr9HdqfK3Z+v495p6wkzqjzQuT6mpnfwgfMOjEZNs+rc3b/CvAe0VZeuL0CjWzUfR9u+8Q1j6iHthGZ/DtoLsSjntUf+hnVTCt+en4wzsOFz6PsJNLldU1Ve4NY0Ul2we74mTGqohS7GaYN9vxUttCoBpWmeJlSV713d0IdVBEBFx1fGt3hA/wf/lLsFAJ0xiN66jVRQ0skKquTd99iRg4TknvG71qkilJxcK6lnjhAvotgR2SPg9jhcAofQYxelKxgrCTartoqXKYKKySm5FNgytRVLkz3NJz034SC36tehqMVru/3kug4AqzByMqINamaeVsiB9Qu9v/d/2Je9rzVl87QRtPjrPtaqjUkSEejdzyaDwcDrjiHcqV9F6rHt7DY05N3gp5ld8TnKiXQ663dy+kDpaALFz7iXmr/4RmVrqB7kOeMPpVL+1Ub24XUcs9zLY2dfuqD97U6Vt7Juok/nawDY6azOvMhhPnkWudqSmutgivkhcqObF1QMAInVe7PU1bJgbYTzSFfCmOHsiSkolOYvr6DTiGmIBjdBykFGKbMIErnFllHelcjd+uW01R+gXnLgZtmms/8wzfky2Vgo79L83Kk6A8KWZ3p3VK0YcHmSgsmK0jQ5uh+dxNZfPy7RvsLlZP/fcwA4pWoawKkJl97XoMSN6sgTBgFPf7+NffYYTTPofNoOgz7vQ+oxSNoP9kzYkG8R4CttjEXVthBZXfN5ZE2H+37Ky+Oyw4n1WuCXfb9hc/mOu7PdmiA+LtKi60NotF9zsm15AgBrhaacw+1nM7g8GC2YKrsdcCfuoWf6DwihLe722vcSA8++xRL9SK0cYeaksSZnavbDomjjagtFv5d9fRp5NI0s3sXra8YtZcORFNJzZJAJP2yZ8Em7Qjer+RZz0WvzJYtRTybBvhm3zi6ymiNJWTw2ewu1xEn0igCEt38B8EkHxK4fSc2xF6ilXeQhuOejUtOo7JBCoyuY9FwH7Sf/49UWApi/7QyAN1ylUa+QY3cy759TAKRk+wolbE6V1xbsYsuJNNrV9FVpNxn8/SFNX32UhTvP+rVl9K97ePO3PX7pHp9G+R8CHgFCYasH1nyaRilZee29buJyNh1LJTnLjkVxEhIcTIzzDM11hzlV4TpMeh2ZQVU1UzBbFpzZBn+8kOf0ryCB0JltsPo97Xel5tpn+o3gLODldqoEE7/0U7D6fc1fUrdXtI/H2ajTBnMHa99Gt9DBs0prKt4JXSCUpnmazSV4yTmMWnUb8ZurHQKFLvod3KDfQqd0bXJ9otbdvOm8n/VqIzKCqnv3LW9RaaM7wJkju3zKfNs5gBP6aiSoESxytSUi+Z+A2+NQBT3s77C6XNlp+ThsmubIoynjy6wNVzI5B/5GTS3cxMXhXnnX2dJ80j3CpH1RXYut4znncOpbZ9LT/jaHKvZiT/i12Gy5xL8eR8ah9d58zbPX0FB3Ap1Tm4h/YXqPaCXdu11RFF40fguA05rNU4mvcId+JbF7ppGzWRuA2bJ9NaIuFL3TXxgQLE0kL5icZK2PhTk1p7H20dEcWj0v4P1zM9P4yPABjRs0YtaN2/hNbc9d6h8+eeaZ3yD31C420RhCYwoty2g0E0wh2gjnt1sJYqdaE5N7sQSgbZdbGGR9hocNv2MJQJ4eFF6OCCWHTrbVVM8I/Pmr2rKIJIMVrmbUcmgmmcfMDUgwVGaioz8ZIphHHMWbOEiKRnHkaScmO0u2OLHrn1XU/3s4Kcd2sctanuNqDMeWzyx44UxSunjOsTFvMr7qUDK97W9BeCX//KYQaPMAVGyi7dNppK+mkd09NtSbfUOj53+W6AyQtBf2/gp6M1WigvlgQAvvZs+kPpCrn+Nw8UgXTSv+lewJ3K9fgqq3wJ3Toe0w+nVrj63lUAAUt08zo15HrjGKJEMlKivas3SA/VV+dHWmWUUtz+mu75FGKN3q+wuqPASb9IS5NY2igk3c0Kii1zztlo9Wk5hp4+7P1zPoy7INgnJZcs530T4x03dc4MrvNsLtt8io1+VF5wPtdzEL179u1+Z/CyMGAFAhSPH1aZS4B85uw6WKEjsw98xlL9bCRXLhSKHRFczxFG1QkZGbJ+WNCdNWKfObp/2+4yyj5m4HIC3XV3BidbiYtV4zEfKxd0YzTztf0+jN3/bwxHdbA26jR9Mo2x2FaPrqo/y+QxNsZVkLdr5pc2g+mdrGRdG8WmSBeQ7ra0GFepgzT9BMOcLO2NspH2qi8tedtLD1pzaCPRs2f5lPaFSAICi/ynD9XlC5JZzcADkp/nlFCVQmharZms8dDCvegnOH8/b3OL3u+ATsdWs5qe7r0vL+wOsoAo+mkbMU1Dz3bljCt8axxIQaqX/3ONII826rZtOcB4crObTV7aOnfjNhCXmq1MI9yMk8tQeEIH7PWoSqcqd+JXPNd5BiqkwmwdRP+D3g9jiyzvGNcQK5+lB2L3gPp+3fd5xnd5ssRTvOIEoQeUgSGMHf3sayz0eRnJTA2rE3IoQgLTmezb9pPoycthyyhZnPop712c+Wlc5htRLxprhi6/g09AteusbAigkPkhjVisOWxpyLP0k0aezNsPCB8w6yhIWfXJrz0c0RN/K582YAVtyw0KesSu6BssOWTbQzng6uzbyk/4Z2GYu19tqL1/wojqPJ2fTPHsXNNl9BZTZS2+1CcZzawkmlEsEuTahnws6BA3sD39+aSR/9BswmI1EijUH6JdRzas/ELWodFrg6AmBLj2dG7pNEphXuuy323CY66vdwpmLXYuvdpG/Be6ZPUfKZYRr1OoZU1BztWkKKd2YaXk6bdKpBUSiiBI6wnTZcipGb9RuIcmnvyfnlh7Ep7Hq+NvSjr/0NrtdtCbg8SR5Ol0q6e4z2Q/lH2KxqJiM9977IwR/fDLgcvUEby5Wf2YkXDg/ifWc/Wu6bxPH9gY/dJBeIZ3Fy+3feJGtuNvfo/ypck33bd5CwS4uUZg71HWva3D6tGt0Kt3+al55PKEXj290/BBgspOc6CA/KEwZ4oiR75Fn/HM+LrHY+OTYnLdzj7qjQIPo1K4fS9A44+Cdc/xo6nYI5SBsDqm7NKZNeYVGHbxkfOxmAzWo9dopaTM7pSdXm3aDP+5TrOIQctLbNGNqW129p5Fd3sMlAiCnPPO2LQW0IMulxuFR2ns5bqMlvenclMvqX3bzxq/8ifGH8s3Yppzf/ArFNAXjbMYBrxy3yERJbzeWwYoJrhmuBf9zkDzZE3LUFL5jnwyMgqufQ3pOVQ3U+mkYrXU357pDW96xOl38BReCJrOdQL37eIrkwpNDoCsZj/5mV74aNCtZsVT03tkGn8wqKXG5/QCH5IqXZnKr3uZLf3rlG+eBCzdPUEqxWeczLPA+VN3/bw5YTacSGW0jILPjBb3W6sBh1zH2kI+1r+UcvAvgpuB80uAlMYdTWneV+4zKqlQvGgS4veldQpCY1H7JQs/9WC3hhO3PzzAGOrYbfn9Gk7cEFOBINjdZWdAJBCLewSoGt32gDCI+wyBOppt0jmn16hxF5D/eEXQUWV1I8mmY5pRDVoOXSAXTU78FiNqPENKSPa6J3m9Br565i+nZu1mvCIlNOgnf7OZPmKM95cBnH9m8n9ofeZGRl0VZ3gLb2TVQ48QdPGOZjKIFTa4ctl2q6JG499R6Nt47hry2BT/JKi9ygitSxzSJKpJF46vC/Xv+VznE1hunprUk8vo82js0kZdn45+cPaLN5FABWXTDL1RaUyzros58z+xy1dWfpdGJqsXW0cO6kXRWt/7ZK/pVupz8nI+UsekWwNKMK6e2eJU0J52v1ZpJEBHVzt/OPexLXqFa1Ast0WrMxCyuOoAoAJIpIIC/i1MXw4bzFDNL/ye/mlzixJs9nzqOGMVodTqmyX1JMGcc5G96cMKE9f1JFKAkRhZuQnY/DbsUlFAwGI1WydtBDvxVbaFVmOXvQSneIqormFNaedY5yIs3HR9/5GMzBnFCjsUXWLTSPh3WGtmSIYL/069CEAqZioqcBBIdGwuh0VEsUSgmip6kOG07FSJiSS4xRE4Zek7uKxom/MVl5jxmDWzPCsCDg8iR5zPxhLp+PfQwAh83KdjXPD+KvBwJfHNGfZ55bW6ct1OWml0501kA4d/ogKSf3F5/xSsMzzsx3T4WKXMYbp/Pl6gKip0FetLTwqpoWiD3feMiWCY9vhHbDIbhCXnp+M9adc7VoaE/vgxvGkJ7r8AbDAXzNh4A7p64rtPk5dhdRwdq+EWGhNI8NRukwAtZ/DH++pmVq0AcAh0Ubn9+96xHikv/m/VOab7e4pp285YWFaJpUQV/3pIlyhIQMG93qxzC003mOwoHbW1bhpma+2lhGvc5rtuShILcXVxIz1x7jyzX+7j4Ko/WSOwnb/DE0vp00EcKXrl4csAyGf2Z68+SayvFyyGg/k0SzUc/eu1aCKQyOLNd8wRaBZnImGJH0BgChRt/gRZ31O6ls1xQRrCWcf3jyS/O0suPKvrOucjwrUvmFRgJBbLiFxpU1O2SDTiHDnS/H7sTpEoS4NYqMesXHtC0mXHsJ3dmqKov+1xmTQVeg36GSaDh7bv7zX1rVywWRkJ7LyXM57D3ra7phc6heoUdhtE2er5mWmTVzrvCU7VSLCsYh8nV5s9sWO+UgPJnPcWB+fnkib8Um5bAW8r7/174qm95Gd4Q+k33ThCj4hASX116sitsRduM7oNfb7o0KVGkNi1+Cw8tg/Sd5A4yDfxZ53IHi8Wl1/nm/GPR6PRZs9HKt4owox2pXY4TbNtpg1lacbMKIyCecOxx+DS86HiQoaQeO5W8BcCxBW52+0/YTpmxN1dXoCDxCjMc0yej2Z3M81c6O+e8TP6YuWxfNvKhjDLgNmSn0Me8gTQkjKyPtX6nzqkEIaugSGaBfjl5RMSkujp46S3pwDXao2kAzO6gKP7uu5ZHUd3x23aJvwkpXU8yOtGKrMWPDYNHu/RA1gyjbKXLPaf1xOD9S03GAgcGfMVE/hWglnSa5m/jM9D4AQaERPmWdEeUYbn+Ks+XaYRG5qMGaFsdPrmuJs37LyZhuF3VKAB5KeY+njD8CcPxYXjSk2upxnrY/Qq5Drs6VlGnlnmZ93aeZ7ujF3o8HEqVkEese8BbHt+uPM+jTv7FjRFEU9BZt9b1ciz50f/QDFjcYR4RRuybO7FQMONCbChfmGMxBVNclUTNpWbF1P2SfTbjiL0TItmgTLkUX+NAvIbIlu4M1XxhOl1qsdurZyFZ8FvkUCcP+of4j3wDQKns1jdJX0owDGExmDEjtywsifif3GjT/UsMT3yRDBGMXehxCz0alKb9sCaxvOlRNA83mjlrVqk1HNtCUrMz0ona7KBJOHSInK6/8cl+0ofz0ay5ZfZctphAtClo+oVGwYsUu9Lzxx8GC9zEGQ71eEF1P0xaJ0/ztfbHyCO+3WAjl62j5fnZHeazcMk+D3oNOD1/fRkJEU1Jz7FSLCmbrqzfwbM/6eZpGARio5didhJgNHB5/E2GhYZBxGuYO0TYedQcsqd4OntrDwXoPA1A+5xAt92jj26fsj6L2estbnt7jSCk7iTAlt0hBQqc6FWhV3XecHhNm5nCSJkSrFa2NM0+l5tJvavEOm/+reIR2gVoKfO68mbWiCVRpTaSSzUrzSG1DVp6fxqDk3YzLHgM75vrsG2TSYTq1Fjr9j5yIupyqfbdf+VaHi33x2jyt7amZ7DMP0Ta8eAphifIKjYQQnBIVWCK094nVXrIxiT0ng7GG6ThKqKEkKT2k0OgK5vaWVagaYSYzn5lXRq6Tj+9tSZhFe+jodYo3ckKuw4VTVfMczRn0XnVA0BzOAVSKsBBk0hNk1JdYUnw+Hr9FPd5byeqDeVFhfojvzWNiDtdNXE7vD3zDzNucLq95nYcxtzZGUfIc+d2hXwUph7TQ9QA6A9XKBWFX8+0XWhF6jof5j8PiVzRzNYD1n8LpLbBqkva/fm/t22CGcrVh6Wg4ukKLunYmnzp33Rth0zTIrzo54yb4ooDIM9H1oLf7xSlUzQTOEzknKBIeWqbZ/qad1LYbg6DlfXnaSBeLe2wwZdnFh9o9oFbx/jaLXCabPqGX7S1+cHXjRDnNdEdvCSFeRLFJrYfqyjuGavF/EWVS+TT0UeomaL4+7Gs0LRAdKsKRQzwV2GqNJSEjMJVjj6lPsF0TPl2z602abRtNlsvAvuP+/rYuBUrqEV7hCx63P8nR5TP+lTqvFtYeTGSTWo/r9DtxugWElrObiHIk0Eynrb7pUg7ygH4RFtV34rw71cA/oj66gkxR85Ftc2ISdoxmTWikGCzoVBvJOm3ltId+K3Wy/qElB6mnO81tjnFsFZqW0S41juDgMJ/yzDi4V78Um83Kg84XyYlpCYADA9+b3kDJPHORZwUMIk9byZzPl82r6lSOi4rkynl6iVEc2VSICKO/4W8aJi1klasJFqV4J+oAzpWT+NP8HJOVewHQeRYwqjWmSpWq9BwwgjqvbmFM5c84FtUJo3BiKEIDyOg2+TDpin8HVBQFR1hLD6vDdrXoyGnnkxLTgdXmzgAMeG8+D3xatFPsTEJJCK5Lxap1CHYLT4XOQIuctbjQoTdaMCkub5TLq52Mc4ls/vH9IvNYHS6GzthI09Q/vT5hjK5cOup3Y1JcGBUXz+dMosH83iSfLlqzddOc8RxIcfCk/XHMivZQCIkox9Qa77E3tHBHuRdL5ue38NvMt4vPeKXjsMLJTbD2Iy3gCVCOTFIJAxTUgrQojMFwYJEWQKXmdZpWEfD2wp0sXr0eb1Td8MpQq6vvuBQ0f0hN+kHyfqLm9adWhVCiw8xEhZiICjaRZfNM6vN2ybA6iHvhd7/Q6HZ39GW9ToHur2guE5IPaBsV9/h6+Xh4vxGmsAqEk0WOWVskWW1oz/X6rZgLmnkagjBj99FKCYS6FUPZfkoTRlYIzXt+bj6eesU6xPZoUgVqKdBVt41euo3wtRapuKI7Wh0heZppOmsqFmyQ7Kv9ZzHoqb3uBVg+luD0g6z74inY8rVPnvgPb2DDlAcAiMg5gUVxYFdMsH8R5Y1WctzuR7acSKOqkkxVnfZ+Kso8beepdOJe8HVLUT5hDfcZ/pI+jcoQKTS6wgkx6300jTKtDq/ACDTVTs9L4VhyDnan6g1padArPpLsKLd5mueBZTHqL3r1Or9527cb81bJDkVdhyUy1vu//fi/vL+tDjUv+pcQHHvrZm5uVgkhwPO+NeHQBDEhFaDT/0BnpFpUMM9Fvg+KHp7YAgYTdHhcW4HZ9g1kxmvO4hY9r710PREqNrlDbxvMcHy19lDNPAs7vvd9eO78Ac5uA2c+4caJtXBmCxxYrDnKPrZGS08+BMsnwIBvtfYd+APma2rn2DJh3SeaNlOvCVpaVoJmxlYSv0lF4HQP2GdvuPiIKd+5upMkNK0ts1szY67pDe4x/EVCuTYAGIPCiVVSOSWiSYrQ7KpVWzaVUzcQp09mv5I3kWl77DOOqhUxqTZw5HK8fCeWVB3Br9sDm1jnWCrxu+saLC5t9SnCqjl5r6M7Q4iudKLPFYfLbsWumHjX+Ck9zn1X/A6SgNmbmMPHoU9gFg5yddrKonA5yQypof1WVfTpx+mk302w8BUaDTr+IgMNf6MUIzRq//rPjHYOxhCpCUQVoxm9y05iSH2+dWpaQTpTMP+zagLOVsFJVHdp99IWtS6688yMfnR1prN+JxWPLWCbszqO6powdah+Ee10+6h4uuQahM6UY9jj80wv9UJ7ju+gLtm6PKFVELn8aB6DI+k/YCYpBKm7Ao/UdakZkTyOeulrqKKksE2txXX6XZgD1JK5M1szEfzRoJlqZCuhbFLrER0d65MvKbQeaYby3Ol8EyWiYLNGgPI1mrJPrYYNU6F5PGwN7sh6taFfemSlWoSU0DF689AMBh9/HqG6mJL1NA/Hjy4yf81DXzPy3FifNOHWzK1EMoagcOY6O+NwSikmwN45L9Nm5+gi8+xY9Sszjt9AO90+b5pR2FDr9GSvUptcYaKl7hD1dKdJPl30QlDbfW/Tct0TRLhn7pvVemSG1KSROIzpXCGaLqWADSN7ciO9/++wjeYO2+hLVt9lS06y5pQa4JQW+tyBgb9crQBIyipgjBLnNueyZ0P8LvhL819VJyibReYX8jSEkg9pUXw7jPDdP2GXFgkYMNlSfEzTQsx60nPsfgvA/xxLBXwtFUBzWeE1/zJYwJGjmcwZLJozbvA6XW65Ywy369dQLlvrk9/pbqaPfr3foi8ARgsWHL5OkwMgOlRbzA41Gwg2+VognE67eF+BlyMGXcksBaoq7kWEoHK86bgvb0P19tr3329zzSrNeTnlfc2fg/Kd00+ct2JyZWtWF/nbY0+nh17zU5euhLFebcgxSyP49UlilTRvxL3tJ9MA6I7mqqIopYNdZzRBYP4Ia+asUyx0XYNTOuwvM6TQ6AonxKTzOpR2qYJsu8sbfQDcmkZu87T+n61j7uZTXkdzep0OpyqIDjNTrVyQN9JZntBId/GaRvnM21Ky8iZy5XOP0zQy73+8W8vk9x1neeSbf7SXzsYvYEwkcJ6zNqByqF4T8ggBB5ZA9fZEhRhRrenw3GGIcttLz+yT9wB0OfLC3ofGaCFMAc5qTsKp2DSvgiS3ND6/U7iDS9xp+QblbR6Ea5+Gb/vDtOth5k1a5LSM05qdeeZZaDkIur0Mqvtc5pyDJa9omkc57od9hltg4mn3ReKR1TWvGlF0xgAYqF/GYaFNrs1B2iS+vu4U7XV7aZiiTYb1FRuyV63Gn2pr0kK0Y0h5qxltk36iudjHJ+mawOwP0YEfnF14yTmMbUp91mZXYXvM7fQ3raXG7k8Cao9dZ+Y5x3D65I5houNuzjpDediuReuJdsRf9PEGgsuei0MxUVVJZp9a+EQQYO/mZaz9YdK/0q4rgVr7v+Bew3JCFCvJlppsU2uhuuykRGn3pzU3G5fDRiphpJ/n18XoymWRpRcLooYUWcd3prGkixDMoZEAJFe8jnlh9xJz5i9a6bQBsGIOxeTW7rlbLCIIG06hY5DBXwD0sMG9YpaVwGHL/URmab4rQhXtWSEcJXfcmfFJD0yftue36WNJOZeCA20i8GHo/9hX4UatXFUl1C04y039d7TsLob9B/YRMrf/ZRPFyaDa0Jk0R+Kegbc+QKfQIe5r+66qmUg2b9WOf66f4+dv497E92h86AsOuWIxmQuPjBYRYqaB7iTZOcVPhOyWCkTg7weua7eelHt+e0Dt99Cofl0au/ZzdOtyYpVUrtUX7qwbQHXkoup9NaZOhrUAYIGrI8bgSJ51PuJrKn4Vk2Ozs9TVssjw07os/3vXIJyE1WpLfLWbCcqn/ZadULSvk5NqNOWVTAbpF7Gh/Sfs6fEVbVq0pFv6z1RPKJnweveHd5J0LDCHvI11x+lpyxMI7xZx7BdFvxsvN+Je+N0bafiC8SxYVGkD3V4CtHPxknMYACfPFeCbKrIGoGhj2pwU2KWZIVe2OMkUQSRmuTVqPAFaeo7z3f+eH6BxX+/f8KC8OUCo2cD2U+m0HbvUR4PDI3DJb22gqgKnKvLcQ6z9CH4bqfn4fPBPaP+Ilt5O+zbhJAQrDr32DI3L3gmA0ViAe4dbPmSD2sBHoFUoqup1Gh5szosEbTjP7DY+48oUGnkW2wMVGh1wj89pcDPTXTcRZ51Nd9u7ULGxln5QC8hhVczwhG8UaItBz5/XL4Sn9zLROYBUwvP6mTUDtn3HH9HDyBYWVFWw29SMipxjWpWxoDdRwSI4mqy9i3LsTqzCSJKqLTLnFqFV5rF4yc6Xx2RN5ib9RtTctICOW1L6yLf2FU6ISU+m1cGS3fGccb8E8msamQw6UrLzBhy7zqQTYtYz9d5WNK8agSoEQUY9E+9sjlGvqcB6hEelYZ6Wa3dRJ0ZT28+/whBlPUHt3B0+eYUQ/L5TE56EBxk18y2A+F15mkfAt8PaUb7dQO1FqyiQehQqtcCk1/Nq1ljtRefR2EnMN+BxWqHhLdrv793S+Jb3w0PLtd8V6kCjvtrvtON5+3jwCH3ypzW5E2p01FSDG/TRVmKcNnf0NAX+eB6WvqZpKOV3hK3TaxpI+93hmfVGbTWn5b3FntP856swXKpKlcgg4ioUH0WnKKzWXOrpTtNep62cmYwmskTexCfMqb1cghQHDXUnedy8iOqntAl0tNuEwq4PIVjN4ntnVzo+/T2LDd14Qv8zz7ie5Jm0cZw+uo9Ys5OKadsCapPp9EaWm5+hn34ldXWnyRUmGuuOAXAo5oaLOt5AycHCQUNdFrX8hFQRhtNeuIaT6Y+n6bjnjX+lXVcCoRmHCA0vx2fqbWQeWk8L3RFcQkdcmrZ6FfROVRyppzilq0IX2/s+ftkMLitpQdU5o1Qsso4mumN8bnofi9DuZVdENXbr6hOSeYT9ohqfOm/BFVGDNBHC244BWI2RzHV14UX3wP98Vob2AkCXozmarVH5vPrPExrtj89k1cGindL+KrTV54bHv2HL6kWMDXuFZtYvqG7Kovw5zTwhOyebA6IaJ6hEburpIsu7HEizOjEpLq9fsrLGKOwYTEHsV6tSQdF8NpwJa+aXz2qzcy4tz1+LK58WTS2hrfBbjHpvqOr86PQ69NZz7LcMwezI8NuenyUNxtKy6+1F5gFoet2t7Ivu6ZeuKIpPQItACA4KJsUQS3qy21myKGZ/Rw4ug6+wdlf0TQA40WNUHEwwfIEjt+hjvVTs/GMa6mWk5WTXBXNUVCIrt/A+r9ryBICfOrUxSn/jFKxVOmJx+fr7c2TlRXZNOrmfxOO+Qp0xzkGsUJvhVIy063Uvgzo3Isik1wR9zpJp4tqSj7FudfGCJo8pYkU1LwjGfssQdlseLFF9lwNrDhVs+hkwLgeYQuH+n8CWxfiFe+mp20hX3TYgzxepDyfW44l8hiUCbNq9E0QO2VjYeiJNy3fbR1Chnv/+9XpqC6Et7+d4VAfCLfk1jbTJeabN6ROGfctxTdMo//vT407CK/g2WiA0FnqMho2fQWVNW4qqraHneEw4MOLkdOWerLlzE1U8gnfdef6WACo1Y8GoW/njf9cVcfLcLHgcVk/W2u9e5FYFnB9H4Gjyvx8599/AM+/ymBUWRwud28F6mMeJuEJX3XZY+a72t+o17KtxD59UeNVvX4tJz05rNC8u1Z4rqSIMclLImtIZ17wHYf4j3Jz0OWmEkppjZ4O+NTV1CbTKXgmKQo+g/Wxya63l2F185+pOLtqigrUAn7gePEGSUvJp3u0xae9elyMwE3FJ6SOFRlc4oW7ztIdn/cO7S/bTWbeDkPfytFXCLUafRd0cuwuDTkfvppUw6nU4XVpENZNB572JPU6ULaUgNMqxu4gJM3t/e6TLOfow0ts+7ZM31+FCcdtuN6oUrr2sIqrBp518XkItq0dBl2c1v0GgCXEO/YnJoMMk7JqvIqd7BcIQBIPma/6IFCVfaFI0oU2X56GK+0XocsKe+dB3qpYeEqP5H/LgtGkOrS350o4s10zYanSEiKpatLbsJLfQyH377foZds7LEzqpLi0KW50eEBUHPcZovpkc2SVyhF3zxYUs2lWwdoFLQLBJz6HELB/h0tE/PyP9p/8FXEfKWW1CdEDR+pSiKAx3POXdrrhXnIP02svBaDRitGkvn1ccQ0kQkSRUuQGn0PG882GCQ8N4S7xPR/0eHlU1s65bXEtRTEF+EV8Kw+mwEqOk8arxG85FNmGJ2ob/GX7W2hvRJOBjuxjiI5rzadRzdO5xKx30e8jKSC00b6JRW2093+G7pGAstmR0kVX4o+Jwjh/exRpXY07HXk+QPW8wH5/pZE1ID27RrSUrI28yb1KtdLavYmT884HV5db8qJK8mskJQ2l4ZAYddbs5LCphq9KeU6ICKYTh0pl42jCXqkoyu9Q4v3I6j/qetZUGYcjV+n5stboc7a+Z3P7tak62EuST/+VpP/PxjJlFtm1DhDYRr607iyX7FOgMDOnSiBt1G6mevAKA06lZ3OR4iwRTdZzpZ8lNOMi5bb8XVewlZ++pZFq8NLfAbcHpmnlMbm7g0RIvKUJFbw4mTolnpaspj9hHcibE1+zL5XKROL4J1smtvGmHdm3gnAjlNcdgnErRQhahN6O3axN/g7noqGY3DniCChWii8wD0LR5G25/ovS0F506I6o7wp+1CPO4xD0rKZ+yBWHw1Zhqkq71xxb6oxgNBgYaluPMDTy4QWkhVJWmG55h/6Yl/3rdhfFb9EMM1S8iM6FwJ9bCqt0PiSKSuoqm6VLXdQAzNs6E5b3TJjgGcrB8XnSj8tPaETOjg09Z44zTqaEkoHJeMBG9qViz3fyoLhcxShohqcVHJ7U5VT5y9iVbubhFqssB58VGblL0mobHP1/Bouf5fOUR+utXUEvRxmoFRST2OrU2hWoBW3JTQVVxOWyc05XnaLLbH2dWUtGCv07/46/Kw7VFVzeeMTf4CnP2uMcj+SOT2ZznCY0MFm0M3LSf5j5h8Ut5delNGIQTRRHEnZxPzPapJJGn2X6+n26+v5/qh2ZROTKIYtn+nVc7xuKej6iq8Gl/vYqhXnOoKw3PdQhU08gl3OelyR2EWwxUjrBogRI8Vha932JDnac4EOrvmN5i0LFifyLfbdTG+stoA12e42BiNofOaQLOyrYjPGB/lkyrk7vSNHcenTN+h9xU6mydQJbNSXquA6vNxlDDYmoqZzlmuafI+aNHQJlfqWGzQXvHSqFR2SGFRlc4ISY9me4HS7bNRQ/TThRb3uTUowqa/8VhcGsU6fUKLlVgd2mO7zxCI4+mkebT6CI1jRwuooK1QWiu3YUqBLOHtSPYlUnVcN9BTZbN6X3RVC8XDF2f16JEgFdVFdw2uL88CanuQZjBApWaayZtbg2jY2kOPvn7EA69WXNGeMcXUKk5/PYU3PKhtl94ZYjMpz6tN8Cja6HRbZpg6NmDcHO+gXmjW+G6Z8CUb5V15Tuw9xfY8YPmRBs0G/DIGtBqMOCOntaoryZgSjqg7d+gj6aFtOtHWDFRGyTA/9k76zCrqv2Nf3admm5m6O6UFFQUFFtMVLwWtthd2Nf22mJhYys2CgiCgHR3M0x3ndzx+2Ptc86cKcC63vu77/PMMzN7r712nH3W+q5vvC9sj3I71cesjUVMeKWxTOqSXeVNtjdNC49TZUN+NT9siEb/Sha/T/fimU0eUx8bnj+Twp1rqSreSwmpdLt3dWTf6coCvjaGs9DojWQTuyYlp+K/q5yQIyniHHvfGMOwwEu07n0omVTwT+11VFmiXBWkiZPVGTwdOoO2l0xH1pzI1oGRGpr15MvjXC6Gy9FI64Dcd5o65A+Hs2Q9h+mLcbls1bgWFkifdLiPc4J3sXnTur/k2v4uWDPrPZa9cOFBH+cJleNMasV17fdws/cZesu70Wr2gRFiriHk0Le5+7E45RSe0F7BXx4tJ7jJeS87k4ajmM2/S4ZN1L6t++WotpqVqii0pZBkasiQqnhCe5XkYBHvmMdyvfoZpa52pEq1HK2toY+d1dYQa7tfyyvOCwhYGrIi40jMZLp+FBeGbuOHhNNi2r6g38+HjocorvFTUNl0tPQUoipaal0R/6y8lbGuzViyE6e/hL3TLqD7Gz34TrudGa2uY23KOH7+6F+kzjh3/w/5N8AyQmx970Yso+Xvaemm+ax1XdZim2Dd38NpdIb6HKE2I6i8ci1D75rFSW39dMv7IqZNZVkh7aQicog6LbvPOJ44/DgJYTSltFkPlupC0cX9ai2op/078UnihRQm9efC4C28Ip3RbLvMj0+iq28NS9vFZpDkeAUXT/zFM1AVFcOSCPr/+iyA8ILcb7asvvpXon31clTJpLq0+bKnTYkjuU65k+nGUYxRRBbhC8YDxPn24VOTKLRSGBt4nNlxJ1AmpURUaWUp1sHh99WRJVXST95Fgdo6Zl+1uy1VchMKss2gprqcNlIpCXrzAZHIeYMhiqwUNP7ziYmbJKo+GKR3gUk/CmVfO4Osk5TPVqsNAMGmSH5dSULy3OERAcTRd3LdhyuYWduFJ9q9TF54jtjweUwgc8aqPDbk11PES+/KdrULcc7o+18/Sz8tLjr+hB1RgXrZIPfMWA9E1wA44mDzN/ClzaEk1/teDTwP7aw3UI+6EzO1M123T2Oj2YF1dlDlrYuG8ukV9RyaigbFGw6wNNmCvBWACFQCmJZFTnAPGjoKBsM6prG16K93TP/ZsCyLgG6SGueIUIu0hIBu8JIhCLAHv5pHtV/nsTP6sc9Kx6zYLRotfJYL5gzhn7n/aHS826GQWxEt8yuSMiBnIK2lUkqUVuhZ/cg1M7hM/YZQZR5dg2Ksb5WRBhM/I5jSlfK6IP3v/5G9+8QY96oueP5achoF7H31aUtGlwg1TlNv5r5LtoD/z1OA/B/+5zT6r0ecQ6GkRiyig4ZJrtoROh4e2V9qp/5lJ0Ujg+EJQZEkdFNkGmmKHCGZq0+E7f+dRNi+oEGyLR8Z0E1ChknbFOF0cSybyruTop7vydNXRSaIeJcKRRuEQ6ZVv0ideCTSsOGLqBra3UXQ/lAcqkzIjq79uLmMx2du4SllEix5WfAjFW2E3b8IuXvFCS+NiDhp1u6rFMZCVm9Bav10D/jiStgxN3ozh14DH58f5Tuqj5LN4poA2o8UhsOhk8GZAIpDcBwBBGtFRtLprwnyaxAZRuH+m1FPe+OXnU06iOor59WHbpq47QhNWV3UyfJlaCjz7IV3S+hd9iObFn5DTUhmoXNUzL7h8kY+Mw5ntjmI8ozo5+fSFExJjTiN1rguY9bEDMyM3lwcuoXx8kIkSSLnutnkXSQMgmnGsaSkZVCTPogP3efs97oAjHpOow7kcaoiyMfXmx1w+Vsu+fmjkFyylMP9PyErMl7LScBX12zbjnVreFx9ldblS/6Sa/u7wLXkOYaUftHkvkUPHc28j55tct/DXIrcfhjZrUSJV7JUR0bJIjBCHKms4ZjAYyT5cjml9iPqJA++2srIscFQCNUVh2o1H6mqDRj08L9Jq/EPRrYpqhijTEvie0NwncWbVRwurSJHKmdO9mXkmhmEjrqf3Ct3NtlvK81PoLqMcZZwSjuTs6jBzeOO18kqiJVRD0rCeP/nB7PZ+dTYyPaa2loMe+GbFcxlidlD7ND9OEw/qjsBS3WSWbGSdntnANBT3ouUnEO5V2eV2W2/HFu/FWVFeXTb/gY7iypbbOeqyRWX3IThZ9kRRK/6+7nWfgtq7m3Fqi+fQ68WY2/v0HpcVoCsVq1xudyk64XE+2IX934lnouDN3NB8LbIYh3AKencpU3ns7iWnXRr2v6Dz+OEjLGjBfW0fyd2xA2kytEKHZWvlMZlb2HkmiILSnPElqdhO85ccYlIskwdboL/hkwj3YIlZg8CHABvyl+E48rfBaBmb/NBgzy1DdsSDxUlJYiMKYelo2ouWrsNWkkVPHL5mTyc8Bnqzw/T7W5R1n5Z8AbOCkRLTvJ3bcC0sw7ej78w5hxr25zDvPTGctrNoa5SZE3u8/Tab9tAZR4PaW/yrvPA5vC/M4zf6zSqKRSy5o54CNaiYNBGKmWPlUmiSyXUXMlOsEY4VBweOOIWvlxbzGh5NQMTaymotMvK0rtCt2Mjh1z/0Wpu+Gg1APsqhGPJHzTwOKJB4pzkqO2f5NZ466Ih/GN4+4izqP7C/itbjCRMVUG/s+H4J4V0+2XzYPzU6PWaBmqgksn9ZeRykdHSr3fvSGnSEd0yGNwhNdpecYis/HBg1LIEZ2hzTqQGojCGZXH3novY5jqfQ+UN9G2TxM7Sut//ef3NEM7A6ZWdyNai/QdXavw6LoI85Lgust5LdGnsNLNFppFpwKx7AfCYjcdkl6pQXi/bp5O1F57sTqZUybDSz/D7/VQSx6Xqd7i2fxsR5JCzesHCZ3BUbIt8BkeUvA9AH1vldunOUkY80nQgPPz+ldvrEytQw3ned5htDMTrTG/yGKZPgO1/HyGN/0b8z2n0X444h8KeMpsINahT5m4PSe0i+8My5hePipashTONVFnCMM1IeVo4OhFWPnBrcow85uGPRx0opU0pQDSAbpgEDZMOaSIbwx8yCBkWiiLB6DsgsXUkCwlg6a7ysLAoqUYZLLQzgo64VUQ8Yjr3C9LAenCqMpfot0NCNkEDEl0q8/S+IhNp3cew4ydRKx6fKbYFqkWZGHDyCwuZs7lYdBRWHFgzXWQBhfHVtVC+Q2QS1Yesgr9SKKmBUEfLWwnf3SqcQ2PvhULbYDRCgih77iPC0B4zBTJ6CkfUouejJWwgspKKBClppS0t2pDHqMn6eASPYFN15QOl7QySW1ZQ8QUNlpvdkD0plCf14s2Ey2P250jlXK58wz+UWYQS28fsm5d+NtvSjwLAbQVwe+JRNA0LCR3xrBMSk0lvJaKg3zvvQFMkzKS2LJEHtHhd2x4expof36EgfQS3hEQmgzelR2R/e6kIyfhr1NOsYB2GTf74sjwhRs2qIY4omU5buQQz+N9Zf98cSlWhItWU9Pah+lJytr3XaLthmMzztic1I5ucjlFiessIsTVZcCH86LyNzlWLaR/agU/yEKiNRp4+1K8jyyqJGDZNIRDw01EqxK1FDWtZEYtMWbLYaQleAKcnkUnSNwB0D6ynrVxCfFIqbbPSmuy3vW8D94SexWWPnx7F5HL1W/qrezmn/OWYtqZdutqufBEj6xEP73nyCBY/fRYAihHA6juBRw/5mR9yrsaFH4c7nsqErnyinoRhSVwYvIXa67dzRO1Mjtt8J6NCi+gh50Yifn8kDHts2lnavIMUwLTFA4KBxiSletDHKrMLPuvfs6hPkHwMXHUPFf86FIBXrQfxBKKZmJasITXIpAqYCj+Zg/jZ7B/JZBmnvsHzTsFvtcvTu8Vzyo44doTSGKa/iiT/PU2ySyqepuOu6bzneISr5E8j29fP+4T87VFnx6TQzQAMqI5dCAScqawzO6DFi0yWt5XTqZGaHxP/LIS81bSViqlwt9t/478IDtNHOYno5bnNtjli19PcGXyWAbJYfAcDXpxSCM3hYvTxEyi+sYghHVJB0Zisfsl36i0AbDA7oEnRwFHppkVsdopytmH+X2LO0dq7hY4Viw74ur3VpVRY8fycdPJ+2wa9tfgsBwsYGNl2qP85BvmntnDU78NbC7bx1YqWScF/C35XeVplrsjMmfuwCBgagvPndeN4jh46kH5tkpsuT8vsCVf9Gq3p+uQiukt7uUiZyaGsojJs5234En5+LOZQWZLYUVLLqMfmUhvQ8YWMmOwij0Pli6vEeJfk0RjdPTOGjDrQhBMrHLxFkoVam+YSWf9avbLUXfPhw4nwa1TApDypN2cF72362Wh2WVo4W6lyDzyUIQKpDZHWFU59JWZTfefQcHkjbVLcBHWzWRv4r8JPm4saKdD9HoQ/jwFtk/lpcxHbi5t3HK3bV8XPW0o4SVnM1lBmZHuCS2WT1Y6K4160eVPFs7OkxvOPu4EiXa0UB7VCUMYvxxFftZUvkv6BmyBtf70PxdKpTe4hKD92L4g59jH/qWxXu3KvJhzlczfkUVDVNJdb2FlZamca1dhcqQVq2+ZFFCp2CaqP/+FPw9/TQvkf/jAkuVW22YNKUXWAoCsd1n4Y2X/tmK5cPLIjZw9pyyW240gNZxrJEiFDlKc5VBm3HZ0IZxrFu7SYmtq99VQfwk4MiHVk1E/tDZe2nTWkLa+dP5i6oOhLs3TY8h0EqkmPjzp+erRKQJIgkVqGzxgJdbYT56PzhAMlekKRedTAaeRQZTxGNdYlswnoJl0y47m24hHhYIrLjCqUOeKjEqeaG92exCOTUnw9Etv6TogNM+xt9SaprsfAcfUcSyAylWqLYc9C4TzqMgYG2sTbIa+IRK14S6Tr7lsuStPCaaRt69UcvzQcXhaTfXhirLYzi8LPeWN+Y56cDrd/y9Ld5Zgm3K5ORwtURvZV4+FT4/BGx9RH2b6tDJa34sKPu3wDJwYac6ToziQ6yYVkl8WWzBV6ulMZ0qjetwlNMnB54nHX5vKe4xGCUj2Cds3JcrMbbaUSJEkirXwV/6y5q8Xr6hrajLJiGrphsNpzKB380zGyBHHeG/pxJEg+ZOO3EezWlhdS9mCXAz8g6MXQRLT9M8d46tTm0/5Vwxc55r8BIV0nvzC/xTbVpQWM9P/MEYGnaSpZcafZik8Tz2+0vaIkjw2Oi0h3QWJaJuVWPPlWKpahU+jqFGnX2r8dU3awSetN2e617Fo1j4C3mnh8lHU4gdvc9zd7bXpVAV877kJVo8ZSIKMPVwevBeBYWcgkuzwJSLax1av2VwC05DbN9utKzqCTXMjbpuB9cNmlSLXOLDLMUiwjOpbqdkbkeX7B6xX0i3fk7dBRyHZWomr6kR0ekq0q4r17cVkBHJ549mQfx7fGcBTJAkcC8ckZSInZeIIlkYBA1mv7zyY8WBghMQYlb/1sPw3FmBlqojxpd+YYWkulSHnL/vDrOxAsMMSC+tXgMejBAA5Jxx0XzXqyFC2qfhRGyUY2OS9kjfMSgl7hoBwqb2HoScJxfW3FIy2es3/xF9xXcQc5zr/Gof1b4MaPWi2cGkPNqEBFn3mXkPNeNNN0iCYW6c6EWN6l8uQ+/GgMxqGI9/oLzxlUqfvnZvqjYQR95EjlqNV/vNP0t8Jh+njUdQNzW1/abJv06o1ISWJsmWUcgjegk2elobo8SJJEZqK9WLe5pLrL4v4+c97H+47o+7cu8XA+aSPGn5H++THnaF+7ksEV3x3wddd6fTgJMahs/xxpQb8PtxTkFW+Up1KRDNKkP4/HL/jDvdR807LN8Ftg7q98KlArAoNN4eWR8O1NwrbrfhxcOgc/Th7Tz2HKqQNxqnLTTiMQjqMwKvdyrLyMJKkOd2J61DESjJ532D9FxoUsSZE+1+ZW2k6j2KVfWFE57ICIq0dXUZ8IuxF0v3CCmU20ccaL6wnbw1l9BBdpczjqHsFRGh5fw9xMTdlFAydCXGy2iWWEqFPEWH2UvIpuWQm4tdgsmb8S5XVBxv1rPhe/tZw3fxHjojXnQcw1H/+ufvvdJ/jYurdKYM2+KsY+/TM80RVKtzdqe9ILv3DTJ2twEqLMHw0SO1QZS/NQkTEc5kXHB6sJl4CzwbtSakSd/Ws1YV9vdUYDeG9qE9h85GtiXWOrPatEbZutSWJtNd/sh9+KvmcAT/+4RdwP0fK0cAmev7qcy4PXc5o1C1d5CzxqstL8vv/hd+N/TqP/cmTEaZESpb3lXo4P/iDUueyB/Pi+2Uw5qReSJNE2VSxyNTsDRVWkKPGdIuOxoxPhxIAEl0pNQPTTcGKp/3+1LzpghOplFYSdRh6HQv82SZEsVFWvFTL3pkFWolhYyZK4HlmSSJR8wiMuq0LSvt0I2PVz9OSWKcij6xNSA05V4X7tLayvbyCgm6TFO3GbdlQ8TFI98VMR8TjnA/CkgeqMPL+Ix73+oGRHzbGsaBlZ/UXFmCmixvzMtwWPUddxNgm3BUiClPv728BbHr0OUxf3dug1ULwJErIguR0ktoH+Z0f7dkUXM+G63wp7ggx/bmHZ1Ia4VvmckXWzuEL9BsUbLdnqKBVyqNyypLK/RqSlT16Wwex5cxkZiI0mbLh4G8ljBBm2RqwBNL7wea5ZdQKJr4tsLYcnHtV27r2vjI+0k2SZR60LKbOJEzXJpFU95ZWm8KVxKNvkDmTv/ZanlOeZfeMRdGuXzV4zg0nq9+RbqfyQLNLj528tocPtB04IvHvjUtKMkiZV0Ipr/BHCxYJPb6PgvcspcLSnKEFMpFOMF1ALVjTbt2oF2UgnquR/T0nOH43FL19FztSeTe5b8NHT/PTGXfhth8E5ylxCocaRwNnmIZgNFJgAKov2YqDgcot9X/Z+nqXaUHxqIkNz34i0c5s1WIqD19Ju5tBtT5Iz43S2vCC4WNKyO2DpzRuSoYCXYIPyFdkRx2xTkDBWKmm8rR+NMyGFmeYQHgudjaG6+cw4jLjUVs3260kWzuaQ7RyVbed8IL41CiZl+TsibS/zPEsH//RIRK26soSQr5rzlR9Jl8XC4C3PBdRkD6d/6TccVvAWfQJv4MzsSo6+l2d1UVp3ryxIKbWkbBJDZRiuVHaarXBKeozC1x+BkG1OmIGWM41WtppAleUhFGr8XbLqykilGj3w73GgHqYIzo5OUgHeGuEA8iQkR/ZvyTiOlcnjWLCthBV7xJit+/2EJJUkyYtuZ0/dEniBTKkSgESzZY4FJSGT7tYuvjCu+YPv5o+DJWsR5b/6BLZnBe5Bt2QsI8Tm717gfFUINTgaEHrrzlSuUz+POGKv1aeh7TvwrJY/CoZdEplYtuovP3dz2E57kjSD+NrmibAzg7k42w8G4FX9BGoMByMDz+NIaKDCqMQ+91ZSLN9QRXkpJOYwVT+RPK1DzD5JcSK3wPVWH2c9O5PpeZk8rE/kgpInqCxqPksKIOQXY0JPdlL6UDdMXWeB8wZmOW89oPP9Flymfkt3tWWb4WAQDh7uV/zlpRHw0qFN7wvYY4HiwPJVcvVjU+ks5fGEKjKuNEVumtOoIZLbcYP2GQPkHTjjU6JOo6Puhn5noxsmRdVifFVkKUJmfe7rS9hWVItbi11Yd86Ij7QFIaATueSWKCjswBiDGgd4cMSLYOeOuXDkXXDlQk4b1Jr19zdT3upJhTH3ihI7iDqbmiJnz+wNP06J2aSjMi9DqAv3TPCTHu8kNc4RU/WwaHsp6/P+As4bPcgXj5zP1iJxrrDjRVrwJLWfX/uHnKKjrXwsYYoAetF6eKZvdC1RDx4CeImODaos43GotPridHEc8GKPt3mnzxuNjm34rlSGJCz7cx/pF+uua2qf5TtjKE+EzmIeg7FsBzcVwlk2vK2H7lkJXK5+w/FlbwGQoBpIprBBwt+t13/ZFXFctiuazQfaQ5G1jFmwlru06QQkV9OcRuEF5H44BP+H34f/OY3+y5GZEKt0kmXLnBNq7EwIl53N3SKMQ0WW8NnZPw5VxmNPJOH03ASXGnGo1HcMQdRpsaesjv4PRJVK9HoTYn6lnxSPhqbIOOsNTKppD/IT3kOSJN66aAgvn3cIld4QEpCAF8kyRT1up9HQZojgDApDVmDU9SLSUQ8OVaa1VIq8/UdqAzrJbo1a3ISOug+6HSOUyroeLaziVe/Bqa9Ceneq/WKAUupbyxd+KyIjWXbpQbBOOHsu+l4Qaoex+TuY/5RwAiW3E4oPpmGrp9n9bZ1pcykNFhOmaYh7yOwpiLiHXSH+rt4Hi1+CeY8Kz91Zb8OF32JZVqTOOWw81Ddsptx1LRv2xBpPN2qfcnKdSOMsJjmyfZC8jR5yywagHvCSZ6XxuPYqk4xPBE9RPfRul0l8ijBmZS32/atPhDnM/wLuuCQUh2jzuSM2zf1B5TXSEJOu4nCi1i8pMo1G0a25xgDmMQRTD2BKDrpkxuPI7MIyS5SofW8MY48sJrPB7/dmu/O8JkujmkJNolDi8/sbL4gfeeVt7nxJlFJlr59K9vYP+SY4iO054n66WHuwapo3XnfIHXjJfRlLU086oGv5u+OLCiFF3xRab3iFjD3fRPhMrlC/RvdWxrQxTQsFk7uLb2TJe/eJbYbJgrem0OWL44mXomPXRWedxpzOt7Ml8zjcwaixtMTqw4a0YxllLKUWNw/o5/OAfgH36xeQUrWJl/3NL1ZCAR+BBopXcZWb2eK6kHmOI+hyzQy6XDiVxMRkCq1UHISQzBCnKwuIb0EoK9F2KMkNHKl700bxket0lq5aE9nWy9jEBGUu6VQxVT+JKsPJuh/foa+8mzhZGNGrrR7IiTlC9ShQTXcpl9R4F52TFXrJYgEa/v6kZ7cnwarGafooST0EAP8frFAW8GTzsX4EVnNRdhtKbQHjAo9R68hqtK917teokkno3+Q0CsvJn6POpc7vZbeVhScuGlmtSOzBbkdX5rz1AD+8KjI29KAXH6K0Qg8FCQT8JEpeElJb8S/OpcjZsfGJ6qHtQMFZtdVs3WK7fycs2RFxBobHYcs0KSeBPVYW6xd+Q4+ld9Hd3ME2szUJ3Y+IOT6rbhOqFH3vu5i7Uaqad5L8WdBtR6X1N8rqvE2+kd5sZ1jRB822cVkBEjJESd1LjmdZtGwJJ8qLccixDoYN7c6L/P3Jj9FMIr+3Fq+3jps3ncmJOx/gAuVH3FKs01ZyuFDM/We71VYU8nHFBKxV79FXEvxts+Z8H+HObAplKf05JiREQ9L1IvIr/hql0IywvfsbsXB7KZe/uxyI2rSLdpTx+MzNzR9UtVf8NIXL58Oxj0H7kQQLNnG39zGypfIIJYDWUqZRfdQju3amtqGkJsDNn6wRQcaK3dTVk2KXZSmmxCyv0tfIESBJEuvuO4YfrxdZ5vU5j8IVAE1CloVzKLl9431OO6uoaq9wCNnnqS+60wj9zoRUO2M4nGnUlNPo2xsFafbWH2HXfFrHQZIS5IRCuxTu8vmRez33tV/xhwy+XVvAua8v4eQXfmnc3x8Mf1Uhk9TvuVt9n25SLvFO4cjwSh72WZmN2r+7eHck4HugSLONDRfiuKJd66FyL6HirQ1aWnwTdxplVjTLS1UkPA4Fv9ueg096jidWazElbGG4Grwr/pDJW33f5djAo5FtSVYV14euZo+VxTO+O0kss7NR44XN896koRzZIxMnISo9Yj4caG4gzRIB6Dqb5iRMd7KtqIY47z5GKBtpXSW+f4avilri0CW1aaeRJAnFa0/T9AD/wx+D/zmN/suRES8Gq062VzpceobZeCII78u3s1MUSYp8iZ2qHCHIDpdpJbq0SOpgw7rhcHTizKmx5Un1nUYb86vpnSOyK+qny2qmXUKUvwqq8xndNY2MBCdB3UQ3LSb0TRb7e5wgFMsccSJNFjhN/hnuT4U3xsWWiRF1igFMX7KXJLeGhIWy6BlBHvjj3fCAPeAsfRVqCkBxRO4tZjLvMAoOvxlOs+uqJRmOeUg4frR6GRLz/ikmt88mwfArxTZZgcxewrEVdhx1OkKU0+34CRJawcB/CHK6XfOFoyh/tZict3wn0klrCiC9G1TmErKfaZJbizqN6mV6PaC9Td2OaFTXaU8yaYownDOLo/sWmNE00+agB3y0lso4UllDR7mokdMIIKtTH/ZZ6dTpsROOJSnkmhl8ZowiXapGUxVU27F0W+jFmLadyOMdh8isUjVnjMNpx6Mj2P7YYTHt79Lep1ZXQA9i2tEGt2xyqvwLs7Mm0aPfUCbn3w6ARwoIxZrKptXlGiIgu/hn6JwmVXcerrmHb50iDX6vTQQ7KfcuuvrExBmSXRgtZF884biaOKdGm7KFB3Qtf3e4LH+EcL4hvLh4QT+VGi1amhIKxJYMBgNeJqmCyHVJrWi3ZfUCDtv9LNP1o3gl8+6Y9j19K0mrXA9miHlGf3RLZkZoGAUphzDcv5BkqY4B0naOj9/GkMOPQ3W4yKScXz6Pfd8i1xPSqZCSY7aFibDjnSpJqRmM7CJS47dZrRksb4mUHzpbUL9KTE7jkuBNVFhRZ/aDofMIJHVmU68bWKNFv3v3+J7gMe01Ami8bxxFleEgVJnPLHkkV/uvZuHX03ix+hrSK9ciqU7aBHbwrvNRNEWma/+oIk04y6VTxy709L/J164TWdX6HO4KXYzf+GOnf7O6gLPUn6MZl82g377pTHc8jOFtQnXJDmaY/wZJXdMwcUtB3tXHkm+lUq2kcSLPx/AMDSz+jMc3HMF92jtcpIpgiBH0EZSc6JaMHgxQXS7KppPSshgibcaxH8WorNadmGMNppuc9+fd3O/EL9kX8LLrUo4KPMkP6pEAbN+wjB8dt3GB8ihby6L3WDX0Bjpkx5aeJSenxvwfUjwY/r9eIS/gTGefld6yLPlfjAn6lyRK/ibl7mv8Id5auIszQvfjat2b4f7nyZCqqP7lNV5wPI+jwYJOczhYY3bi/tA/WP+zCAqtMzuwbedOPI/nAKCGanBLQQJKrGM/FNeaDWrL/Fs1JXvZO/ctAJ7QXuV0RWQZP7Q2iVMe/pDSB7oQaOCMtkyTke93oZu1G4ByK55t5X9slmNzkK0DCwrVR7U/FKFUePT7zRF12fqOl7X7ms9W+dQ4nHd7NsPVlNJRlFYd/zg1uoKDEAl4qbGdzpoiNU+EXQ/+3mez1OxO6ZWbiG8jSmo/XbEPDr0Wjn6A2nqOHkVqXAngcjSenxNcGilxwhaLd0VtujcX7o4EIXtlJ/LgKQ3ekWAtLGxCtCK9ixCVARhyyX7vCRA2+PrPxd/Z/eCQC6POp/qosgOb08+Et09izoD5LDy6nqPOVk6+9djueBwqHyzdy9XTVwJinWNZFnPDHKV/AmpqxHdgkvo9E5R5ZNRthVcO50L385wZnNKo/T1fbuCf37VQctUEwrxTcYixLGv5EwDs1jpTXO0nGHmPJJZ3mUw10e97WNioKkFkdekOERiZv7Wxk7WhgxHghUXFHCmvFsdaMjVqGjImLziep5+0MxogOPQa6HkyuJJIcKk4CRLsMo77+oiMVCvkp6u0j/zdsU7Yo/81n7Qasc0REHaC6a3EK3kwJTVG7CaCkE+IIh3bcjn4//D78D+n0X85Eu3soHDNsipZcPQDEc9/fYQnljYpYgJTZBmvPVk4FDlCftfKVlpLcKlUeEOsz6tq7DSy+2pIGli/PK3CGyTd9pZHJDwBWbEnrO9vhecGwpbvcSgyQcMkoBuYbYeJNNxFzwn+n85HCcl6IINKoTCW+2uExDoMhyJTYEW90ElujfeNscj+yqijJ+xMkzX4ajLUlfD4TKGGFmwYAaophCW202jfMqGKNm0cbG6i7En3C4JCZ6KYQFM7Qp/TRYTGlSR+NA+U74K0znDELVGltKpcUTp32E1RYzdvuaiNn3FFxJmVHu+IZEX5Q2Yk1bjYSkaqRxSegp0JYE/GVj3j/SNjNDvM7MbXXw8VcZ3YYka5W9bEj2rUxuNQuTp4LUp2AwNDVplrDuB05Re+dYpIvRqfwVXBa+li7o5pWiyls80lDCIrrQuTrdsi+77x9mJlXWxNe4ZUxU3666D7MGU7CqMK8uKE1EycLjfJekm0vhKoKIqNdld5Q/xrVsNIDSTtm8vxylL8TRD0euyIrWVZ3M9l/GL0ZqSygSxdLAJDiqvFyPZJ/q85Rp9H37KZzbb5T8KV1sdkNMNV0UfezRPaK9ThZrouFp+hUKzTKOCPZhIlqOI7EKwqZpvZmunGUWT1HRPTfnD1bNqW/YJkhAi0GsSY+M8ZbG1gaOnnGA7hoDlMWcfImplkBPNQHWL8GrX2ziYzzSqSe3Gh+/mYbWEibKtB6vM4eTlD5S3kenpTajVh3NaDIksoGFwSvDmy7bhLH+Tso0cwoPZnehR8GdnusB27lwRv4l/aVBz5S5FqizBcqcxwTmHkihtItcrRHBq6MxnFDFItJwPg1DS+MYZTaKWw/RwRVXU7Vfp5KthQ7cGf0p3p5lh8VgsR398AuSafasvD9xktLxAk3U8nuRC1sgmVOd3HT8YASjJGNN73e2GaTXNv2AjYJZP/UGfTigr8Ffkcry2PaeO0osZqnSTG1JLkQdyV+BCnmY/hc2dQUeNlAQPRHC6GWmvZltRMqUo96P0m8nnqAS6s/g2oc7cmEPDRWcrnfUVkUJZvms8m1wCOT9xBXrXBGYEpPBiaSCC5Mfdb/yPPoOzq6MLAVJz/FsdNCI339bFUaxn7b/wX4RrrA+Ilf5OlYd8vWoX2/U3sNLNwuz38+qgoA7pMFTaGosQu6HoUf0uZlcinxhHcr70NwD/0u8mz7Z48K42MCS+w3cxhY9ywmGPVbmN4LjS+8QXel0TZDlFevWb6FHqtjS7K8hFz8BrXZSxyXUu6WUJZWezCM+xsecHxPC/pJ5Mq1eJYOe2Ans1vRbiM/A7bxmgIy7LYW9b0nNzvvh/peIfgdsqtiLap73gJO1eawqOhcyimsX2NZcGjbeGRNrDyXap1CSc6x6bkUWaJ4KlDObBMox6vlDI5eC2JS54g2V3vHcgZwNULNLYURuff+uVpYTTlCKiPnCR35O9dpXV8v74Aw7TYWFBNx/TYDH56nAitmgg2WpbYd9hNTfLtNImqfeIHxPhw+K0QnwGLXhCKc83AFSwjfp5wxrygnwJvHgeWxYTBbakN6OTVk4zXZIkdJXVc9NayP43vSLcz0gOWIJyW/JVQsIaPvZMYJjftHMqvappKojm4q3ay23UulcTFZnbrAR589EG+e+thkqkhhWom7b0DCROXJrPh/nEkuTWyk9xsTBgBziQ+qBaf378mNOY6DAf0HfWC7p867uM2TXDjWkhUKBn8cPMxBCwNtxREUe3vx4o3hcJ1bTGaIuEkRKI3l/tShK2rWUFmOW8l+xORZT9S3cIVyld0lvLoXyk4uRwh8S6HTItdSgemZL1IflITnIy1RSI4v3xa84p7/8Pvxv+cRv/lCDt6MhJEBHxl9gSoK4G6skZt/fbE8sVVgqhMVSR8QQNVlpBtB8SWh47lkPYpsPUH2vx8Ez1aJfDyzzsiGUcgOIrChkJDha76mUb+kBHhCZLqlX5p6Z3hnI+Eo8WVBAnZOFWZoG7iD5lk+7aDZE96VfsEOXTXo8X1We1EXb/ijCVfQKTpPmJeEJHMdjsUKuVULNnRqG0EqpPlNndFsGEEqLZIlIoBvHMyvHakkA1tGDG0yeAoWCMU2Uq3iTrvzy+HcQ/DcU8I5QnVKY7NXy0iLooDDrkIep0Ce5fAnPujxNt6IFK7HLKNmbR4Z0x5msehkICXTKkShxGN/sdJYpGeN+qf7Mo4CoJRA+NM5WcSpJYnrxotnZmmeIalViK/pjStnvL5w9cypEdsacaS7IkR3ougJT5DVZHoL+/AbDActbXyubdaKG2orgSWGdHFSFuphLZSCQ3Ry9pGh53T+bitTTbsjmPz8Z8w+PSbkR1u2uu7WfdFlJjcX0+OHeDnRQu54JcjG/Vr+GoYIO9AL21aTh3g0ZmbmRPsjZklJmDVIwzBH1POYW/CwGaP+4f+KQkOcASbyLz4D0SulM2doUmNtvtDBocF/kWi5MWxdz79HAUUWymNOI30YNSJ1NsvooNeQ6aCeN50PEGbyliSZEtWsUyd2annUNTpNC5LXMKF6g/EGdUEFA/bzNZIgMPworkTUOspvHjroqVUq6dPYe2Mf2HWFDOYjTHnUDXbaaTEOlrGKKtwSiHa6rtIPwBS12e0lxgn/Rr5f3CHVJyqQqvAXtpWrYxsd1piDJmsziBTrsL01/CL63DKUwTx5FKzO06COFxx5LYbz0OhifiVqDH/WqspjAi+SJfuUWP+n9ILvFV1MYcUfsIXjntbfJd/C8xQAAUDta5lHhHZJn43g+J38P4MqmznbdCUybfSCFnNjMe/AyteuZSl/2peUtyvW9RYYrG0y2qFVLSOq83pMW0kxcESswdPhM7CdCbZ12xiqHEUKVkEDZlSJZ27PGLcypez8Tn376AYd/rFnHbtU7/11v50jC14hRe9N/OY9iq3+UVWgRWsxa8lcbJvBvF1u9Ekg1wrk2B609kqaRnRYMTilFPYnTysyXZ/JuSSjVymfsP6tGZ4Vf5qWBYOdHzODGpl8f3dXlxL3p4d5G/6lZySBUxU57DZdRFOc/+LStWdxFHKata5og7IW+K+x7dnObWWCw2TnA7d6SLn0yUYu3jtmu7msrqp1FY1tg23r5wHgE9LIVAvcDJXPYyvjAYO3gbOwBqfGMu8lpNn9dP4SB9NfN4vdPG/Q0d/Y4XMPwL+kM5H+mgUs2mnwGcr8zj8iblN7quP+vaqbliossRdx/ck2AI59LfOOxi/5+HGOyrqKbmt/4wq3UmJnMYvriN4QP8HIBbm++M0qvaHuEb5nKWuq3GsnIbUgPj323UF/PO72MyNhgpo9bPum0K7NBFE9dj2eaU3xIZ8kV3VPq0B1+Dmb4VYS0NIkrBZFzwFu+c33t8UNE+UOmPnPPhXL2Ev/3gXrKlXvqm64YSno//Xs7kn9TQAC4J1pMU7aZXoYtGO6DutKjK5tmiPb3/8VL8RIZsz0SnpdJIKyEuIzsPTHE82c8yBOTpaJ7v5/lQNaVE0u+ua0DUUWckAKPkrGKcso03hHFa7Luda9Qs6VC3FQkaWpAjJeeeMeFZ4s+H6NdzztQiUjuzcWMpeswP6314zijk3ibLj+rb6t+Yw1rsG0D49nmpbJThcPRBx3tQU0jUzgQeky9EGThDvBGK9AbAxS6whBms7OFJZTRJ1BCyVJcohOEPivctrP56XPZeRLPtibMToA7S3fX+roC5pYo37P/x+/M9p9P8Az07oz0PjxaBVnNQPVn8g5CwbIBxJCUdRFFmixh+KkVx0hhWFKveibP2ec4e1wx80IhkuIJQXwpOUWs9pJNTY6hFhB41G9bIAcuVuWP+ZmDxqi2D1+2Iy1U3a1q3juEVnQaFdM+tKFpKR75zC2rS7uL9nnnCsNFBOCyMkO3nbEAajU1M4Tf0Fqb5hEZacbG8bQqoL04SsRCfr8qpiJe2dCUIlw7IgZ5D4aaiu02YonFhvcms7DNZ9IibZ8h2iHC2rF/Q9Q5StmYZQcds5T2Q7bfwSvGVRg2OAzVlghMT5gZDNCZPs1vDatex+W1Y1ETviEZcT2b7basUg/1RSMnLoWPITUjDqUEqniif0s5p8dmHkzJnMdeoXor1UzZHVM5ps19BhCOB3iDKe2cZAVpjdxWNUJC5XvyXOii1r2aZ2pUIRCy63r4ANysRIZshpyi+MUKILe8OITv5+y0GaFS076zH0GBRVRbEzTLz5m5hv9OUj8yiqpdjskPSSX0mVGpdNmL5KgEb8OwDVljCijlx8EcudV6CktAVAixP3ujdxMMWOto2OC8NBEDOxHQnBMuoKtlKbf3Bpyn83pFPJlcpXjbb/vGBuJKU5VFmAU9IZI72CLz7Kh+CvyCdYJdLG5xt90Z3JAOxJGsJCow8ZUhWqKzbSackKkqmTp7YhENeGLoENpEi1SKqTUMexFFippFNJvFmN5klETutEteXhZ6NfJJNyZ0ktA7Y+S7/V9zFqxgiuCsQSQlpJ7ZgQuIcV6eNjtoczzw4/5RJ2jntnv8/GLQW5RW2sniI5PCi2M8WyLAqsVNabHThcWYchOzGCfpbo3QjmCAL5HKlMqKW540kOFDJC3khlPTWqDy4dxup7jok5R60jgwTJhypDe6mwyXf598DQQ8RJAU7Ne7zFdgE7w8nUQ1imgcMKsnu9KGP+MfMiusp5ZO374Q+9NgCpYDXvlXVrdr/PUpmqi4jn+OCD+DbPoSg+1gEiKQ40dNYnHEq3wDr2rJlH+t6Z3FXzEN9xDWrxOoyizZwqzQOgg5nLcXlNlG78p0FxkmjVIUswyhTZV3Wmg2pXDpakMqRyJh84HuZVx79Iqt2xn85gT+IhFLoOQo3yD4KpB0mRaskp/3X/jf8C6KEgsmSxtvXZPO89hl/euJWlz51H6zcHkfPROFQ9Oic6XcKh+VTojGb7y+o2pNG2wdYG2uz8hGIrGf1q4XCf3eUueh17eUy7tEQPF6o/kjvr5UZ9eDUxlyVKXr40RzIzbjwAOT2G4uwQ6/xrqIroLxZ8PbqksMl5EausLvSo+ZVh8iZGy2v4I7Ho23f46cXJ+C2VGeZI/hl6rMl2+yqaz/wNwzStGHtVNy0UWSy6/S2QQ2dJlSQGmyh9CtSzKxSNUjWLq5Knkt1tKHsswf2iHUCm0b5yH+lSbHnc5Ud0igSGgRgZdssSdl//NkkM7yQyoBz7cRoluTWePqs/824ZTY9WCdT4dWoDOu1SPRHBnAg8aYKvsyUcKDmx5hYKwhChnKCuFBJyYPQd0XZj74Oy7dDBpijI7g8ZPeC6Nbi3fSO22dx62ckuNhZEAzqaIlFYLfr2tcTX9DtQndKHxUYvAFLi3fTY1bxtEF5ThA6QXzMtmE9q0WLBuwoMkbfwmPYqq00xnn65fBsnKksYrAuy/xHyRnRFjB31LfJ+bZJYmVsJ7qiyr9yEzR5OCOiQHhclTLd5GdeZHfjeGMYqp1B13qD14+HQudFAeQe7CkH3c2SPTDZe1xXF/nyrRz/Ibvu9z3OLefkG6z2GyZuZqM4h30rjjaSr+dkj7JiE3bPoKBVydckDZBTZlBqGLhxEIKovwvjhDlj2+gE9z//h4PCXOI1efPFFOnTogMvlYtiwYSxdurTF9p988gk9evTA5XLRt29fvvsuVgbUsiymTJlCdnY2brebsWPHsm3btpg25eXlTJw4kcTERJKTk5k0aRK1tX99Df3fASf1z4mUlJ285TbwlkYH5Prt+uUwYXB0cavKEjV+velU1oTsSAZQQDdjytPinWpELjHsOLhuTFfUhk6jkNF038WbYN3H4LIX9KVbcagyumlxVK1d+tX7NPHbnQy+SvBVkFi3izYJkihXO+WFJp/FBH7gQ8dDgIi2vKmdw+aT7LKQ2/fC9UJJgGFXAELpKGiYJLsdvDp/J8v31IuoxGWCGRIOoAnvwXmfNc40Ou0VkWF05WIRRUluZ5fA2eppK9+BLyfDpq+FAkXHw6PqaUMvFU6otM6CcymrLwy7DEbdAPFZrC8W5zFs/o8ElxbhoPKHRCpqghKg1nJR5ekAwE0fr0HF4GxlLukfHkdVUk/ec0SdRDlSGROV2U0+uzAG2Gmjg/0v81ToDNoHtrTYvj5G5wqDNFUNEC6b1+wsjhlqbPR3kecocl2i5lp1uNAkg9pcYWTmWWmcE4zK6QYMiyuD1wHQXi7miLJPG53bmzOSlWYXNiSM4vzQHTzrnkypM1YiXdbFhGY05FSpE1lNDVWdgrrJqMAz3BK6jOHyJtKlair7XMRjobNR0jsDcEr5NHrtfqvZZ+K0QtCqN0usnsS9MoT4V4fH7N+37Y81rP9sxFte2solWJZFaY2P1xfsJGSYjJt/Og/YJROh6iJCsotR8nr0uqiDb88zx7Lvw+u5MXgFNXHtkeySDVfpBobIInqqNCC4L/Z0o8TZjovzptAn/2Ms1U6tV50cfex4ht0zh1nWEH7UB6GmtEOTTBIlL/fqF+C1hKH9wTO3ALDRFA4shxUbLdcUiWKSKYqLVYV7yxjHW/oxuOKT6TTilP0+mxlxZzK/9wONtkvOeBTdzrwxTMYEn+JmbiBoKQQUD2bIxx3FN9NdzQdgtjGIJ/QJOJJzyKxeS3c5l9db3Rvpz+NQSfLEGuoBlyC5lFUnAZyEastb5No6WPgcqawyu5AWaJlI/63U65lhHEpAdkVKwiqTxHNNq95Ea6kUqwnOvYNFacFuKkvyI/8PkrczRmleNcuoyOUWTTj0zlHm4KjZi5EVW3ZRlH0UHgIcH5oDwD6/CzPkx5Ad6AiuBWfRao4JRbMYwgb7fzRUB7JkUSvF47CJsH9NO5Wf203GkDU8RnRR5jZaJkIHGFf6Nv1y/5wsk4aoKiuktGA3AIadAdCh/K9XbmsKId3gC2MkGVIlx3u/ImXPTIbJ0SwRq145ZZgvbakpvivh8t76aN1tIIVnRsvji28S46zDV0zIkUhOpnAsjz3vVjp17dXkNfVc/0RMCfcQ/0vsThXZ58VSBkndD8N9ypN08E+nuO2xpKTElmJ51dhATDib90uOQpYs7lLfxymFeN/xCG86nmj5AR0EaitLOGTZjRxV8i7BqiIeUqfhthpnZxVX+/HZdlJLcvLekBHh7wQwTBNVlmybN3qc1UQpjEnjxXeMza04CPh9nKLP5Malh7H7buHsOxCnUaU3iBXu/9rVAIzulhnDC1ofhmUR0E2SPQ4yE8Q6oD4dRHM4bVAbMhNcDO+URm1Axxc0IplHMbh1Bwz6R8udKS0oRNTHoAugn22Php+XERS8pevr2XS9ToZfX4Lzv4Q79gnb2JMGK98V++MyI3b4i+cOijmFKsvUBcT8EraX/2iY1QVk2OqZ5/g/YtjOF9ibdXRkf/13Jkx7oR+Iah7wlXk1WSujgejh8kZypHLGKcJpsrcgWh76lTGCeMmHYSvR1q/o6JWTyLai/a+Jw2s4rd4740W8R33l3XjwR74nU9Nv5zXjRFS3LR6x2XbghbPHZt4B2wQXoLt8E3PNAQB0rainfg2criwghMo+I5VKO/u3w4bn6SrlYskKnfPtNdvyN+B5+/P98mr7aEmcY9cBZrf9DweFP91p9NFHH3HjjTdy7733snLlSvr378+4ceMoLm6ahGzRokWcc845TJo0iVWrVjF+/HjGjx/P+vXrI20ef/xxnnvuOaZOncqSJUuIi4tj3Lhx+P3RQXnixIls2LCBWbNm8c033zB//nwuu+yyP/t2//ZICIbV0xpHWtqmenjsDFH+wI/3cOGqCdQG9KYnip8fg5JNuDQFf8igyls/00hplGmkKRKaIsdwHPlCBiOKP4SvohKUOZRCdZ7IyLlkDgy/GhQtMsntqLOjKWGHkjNRZBWFJbQr9ohSrl5NL+DaUxj5260paA4H5cl97D6TIMlWsNm9EM58C68dUQorx8XUhjvjYcBE4eRxxotrOeMN6FevBGLNRzD/SSjaICIh6z4RWUKWZWc1SVCwGvYthy5jBfmfZQinkTNB3Fv7UeJ5FK2DL64Q6d9dxzKl+AiODDxFwJmKwya284bEZOjXDVyqQoIUIF7y4ykWpS+Ld5bRV9rJrdpHSIEaHHotbn+0nKS1VMYAef9lKz7LwXLXlZyj/tSI56VF2BN5u8nf0OOOqIpFiZXIOjnWiP1H9Wv09InrjksQ0ZCVr4t3JYVa2knRMcTv8+EkxC5TqEGYTZAnag4HlwVvYpNrALeoH3K19AmevAZKGnZqff70q8n76sHI5s+TLybfSo1IaodRUVrIxepMdptRqfUhHdPwDr2GLt1ElkIcfpz+WJ6HTW9OZuuPr2NZFt+Yw1HaDuHuwPmUW/GRNGOAon3bafP+4ZQWtrwQ/zvhSpfgvNBDQZZ/+E9yZz7DrR+vZJPZjs3tz+UG41pqgyYhxc395gso5VGH/09GPzYFM5mpHoknrXVElSqz6GdGKRsAUONSYs63NnM8O/3x9K37FVlWsVQX680OFLUaDYjF1oOeO7hLn0RCq06RUs2H1WmESgTXwmk2oesqO1rnMGOd6mqwirnOmxhdGJuBVBDfm7VW5wN+NuNveZ3jzrq80faa9AH87DoKEO/y6fJ8TlcW4JAMpqVcT17KcHroW8hq143l43/maGUFE5S5uFwuJNVJb2k33cxtjfqtj0CindHljCcoOek56x+UPdrvgK99f6hO6Mwd2q1kmaVUVjVPFNuudjWPhs6hMGkAfkPi1tCl+GVh1B5R/B5tpFIRQfydSH+lP76XjuDXabcStMnWj5OXNNvetDMBvjOGcqf2Adtpi57WI6aNEZ/JHiuL/qZ4F8sDMqq3BEN2oksqRiiI6askqAmj+RfPGHZmxHJw/SdCUp0YlsRmR58IsXdG+Uo66DvY4+pJVT1ODdXdMrcXQIJZhTvQuLz4j4ZlWZS8OI70V/pTkL83orrTFOl0U8idfi3Ve/48p30QjRtCV9PGv41J6vf0lvfQWS4gYKnMN/riKI1mnYYJ2W/VPuS64FU8r13cZJ+teo8iYGlcF7yKzAQXuuLCqddgSPtfuG+7aANXBa/FGxTPx9R1lrmuwl0r5p/v4k8lt/1pJEo+XtGeppV/J96M/ryjH82FwVsotRIjWUlhBGor2Wm2YlN/IUJRJGdwWuC+g35W+8OM1x/CSYhcM4Oq8iI6ywW4LD+PzBZZ9SHD5Lk52xj6zzm8Ml/YOGH13/oI25p1AT3GXtVNC1mWbJtX2IEnPLeAmz9Z26iPWim+0TbaDoV7SmHC+9BlDKGAlytqXhBUCnHCmRfnUPhuXWGM+m1DBHQz6jRKFRkdHoeCL2g06cAyTIuAbuBU5YhjSTsAp1EY8U6VGn9IBHmbWgu0BDuzjbgDVLTK7hdVJA4HDowQnPIibLSzl/1V8Jo9ppbvghVvQ+kWuOg7IZ4DcMs2SBHzXU6ym9MGRpUpVSUq8PNnOY2c+b9SVY94GsBRKJw61ZYnplww/HdDZ+HIR3/i6zX5MduKqxsH+8NZ/2EYyHxmiAysk5XFXBG8ng29bwRiM408DgW/3vQ7Ux/nDG3H7Btj1TDHBx+gk/89RgWeZYY5MnIPQ4NL2O06F4e3QDScNEsEwr+7Rax5dL9QiAa0tdMZKG3jHf1o/ESz5JaZIuvoHeMYbnJ+wT/KX4RvbiC5ciPVWgaZwVw6lS8QNkK7EaDadAPxreCMN2GwPS7mNx8g+h9+O/50p9HTTz/NpZdeykUXXUSvXr2YOnUqHo+HadOaJsJ79tlnOfbYY7nlllvo2bMnDz74IIMGDeKFF0TmiGVZPPPMM9x9992ccsop9OvXj3feeYf8/HxmzJgBwKZNm5g5cyavv/46w4YNY9SoUTz//PN8+OGH5OfnN3ne/w/omhlPksMS5MvulJYbr/+MNN8uagN6kyVkxKVDWhecqhh4iuvJrcY56pWn2ZOTqsioSmymkT9kEGd5RZopwsG0yHUtfHezmCgWPSeybPQATvsaUsKlQ+nd4d5KUB1i0CixjavSrUK9560Tm7ytJ/QJPKePB4Rkpdt2ejXCth9h/ed8t04Mfo7IfUSH3eIaP7mHPwnxmTDtWNgxR1x3/Zr+nx8V6hKfXyIIAUG0bzcCxt4b5VKSJPjpYTEBZvURmU7LXofKvbDhc1HP3X4UbJ8tIiy5y1hpdeM4eQlW+a6IhGYkgmaXp+1FOFEcNcJw6pwRhyZF79ddl8t4bzSCM804tsnnFkbAdpq4JWFU5kjlB+U0qpPj2Wa2JikpKcYAyZCquVj/MKbtTlcvVqYeL87nEROwhjAkPFKAx7TXIm2DVYU843iJ80PCMJVdCTSEu2A5y11XMrr4Xa5Wv6K/tRlXeWyWlGmarDC70nbXx7ReGa09lyt2cm/oQsrju8a0ryncwfXq53zijGaPZCa6uP+UPpEUcEtzI4VinU0997xLt0U3seWD27gpdCXxcXFMkafxvH4q/wydG2lXkb8bgNy8/xynUScEkaXfV0eybw/3a29zZt6j9JT3knLktcxRRrHQczQfZ15LSNJisrqOl5dwvjqLj+U7Wdl+ErMyBPGr5Kug2vLwqXE45AyIOV/Xml+5ruB2HISQVAc73X2ZbQ7ikAFRHqnznfPY5vwHqR6VpLQs8i5cRmelCL1aOEx7yuL5TlRFBslWuVPMOVSb2NFlxDrbb7r+Fu65s3Hm0MFCT+/FbPVwvl+8hqueeJ2nHFM53BJGZomrIzWWE5cUIj4pnQy3RWupjI5yEW5VQtHEviOqmyDgr4e9PS9lVOAZvO3H8GLCNWw3c1iud6JmR/OOlIOBq3Q95zoWUCRnsH7zFoo2LcT7YJtG7c6qfIO7tfdwV+/Cb6qcqfxMWt5PACg2Z5v5BziNCq0U4sxahu99hdyta3hTH8demif5N/QQVVYcFWPE9/5V4yRCHQ6PadOucDbjlOWUdT2LDzmGk346Gmv7bJFpJDkw9QCWvwrddhq11ffiMfefefN3x852ZzAw8Aofpk3mAf0fmIbJoJIZdK9Zwo8ZF7DQ6kOhJWyKtJxO++kNTMURWxb+JyBv1xak+5OZFhALzPVL51KZ2o+P9NEiQ/gA0Hbr22z58o/LhmmIYF05D6hvojli51CnpLPdas0hVT9wYuAh+vtfjezrI+1mhLyRhfec1Gy/wwIvMNMUJSMr5D70CG3EkPfvNGqbk8135nB8urBLQrYt0754Nv6KAgaXfE6b4A7StQDjlOWk1m6DzF7cp19AZ6mAdKkatSC6WCvasIA+c87H1OL456kiay8oucizGnOn/F4sLY+n3Irn3NCd5BaL7FULiS/XC+fknjIvTzcQuaj2NX4PdDvLqjYQOwaFOY3C2fUAG/Kr+Wzlvph2H+tH8GT6Q40vsLZE0CqkdWZxyinUhOzlV1rniB3YOTOe0toAczY1r+wV0A0+MI6K2LEgHADeoBFR0q2PtfuqeO/XPbg0JWLLa/spT6uPBJdKtV/HG2ymMqAluFOg96nQ+QAd58teh6+vF38fciGc9rp4PmumQ02+cDwEasXfAIoqKid2LYC8lRHKBnbMjXB+Ajw9YQCdMmwbUpGps8vSfA2cRmv3VbKl8PeP12bQRy9J2Nw1SjILzT60kirQLZl+gddj1hxhrtSGQjt5lT6+XVsQs+21BTuZb0SzX62UDjH7e/mn8ZU5kg/qZSHuszIo6yDGivrUrW5NEY9TjwrpNAVVkemSGesEHSpv5mLle647fQwWMrtKRSBuhFdkDEU4jdI6w/ipwqlXnS+cRhk94L4qaDOEc9WfOF+dFckoL5FSeVEfT4mVxBxjEG0y09BrywS5NWDGtyIYVn3cOddeb/mFA6m2UFCVhOlAQnXwQuNy3d+DD5fujfBh/X/Fn+o0CgaDrFixgrFjx0ZPKMuMHTuWxYsXN3nM4sWLY9oDjBs3LtJ+165dFBYWxrRJSkpi2LBhkTaLFy8mOTmZwYMHR9qMHTsWWZZZsqRpAzkQCFBdXR3zA2IR+Z/8Y1lW5O8frj8MlxTCPP4pzOyBLR9nc7jU+EWmUWTfnsVY96dgIWEOvQynKrE+r5p3fxUD5G7XubxcfB7eoM6HS/cQLpGVJVEO5gvqkb7qAgaS6sQydUGiWq+e1sruB7Pvw9qzCHPIJaj2rptDV7Di3LWY8VmY4XvL7o/VfiSWJw1zjCjRsLylTd7XarMzT9ucPakeDacq4w3oje+/aD1s+oo7Pl8n+rO98RLR5zn+xYV88NR1mPlrsIJ1mKoH69ubMVe+G2lTH9Znk7BSOmIOvxozIRuz7XDMVv2wPOlYFlhV+zCrCzBTO2P2Pg3TJvu2ZBXLCGH2PQPLNuLM3CU8o73ArdrHWJW5aIqIItXZ9+IN6jhVmSI9nl+M3hjBAKZpEjJMHESNIVN1gRHENE0Mw2C6cRRBS6Fo82Ly1v4knCifPM7aeZ9jmialXpNzgnexpl5W0JaU0Qf8PgYthV/MPiiyHLP9PX0Mu5SOMdtqtDQCWnLMszxU2YhpmowJCEN+28sTME0TX40oG1zgvAEAA6nRuQ2bKK+1TziK/M40rLrY92Rl+ilMCV3EKrU/H2bdGNl+adEDGMjUKskx7evKY53QXxqHNn6XVDey7qMufxP6A1kYuo5hE/1aW3/kIfUNEuQA56uz2Gdl4JaCkWPLqyr5xhjGNrPNv30s2d84Ewr4CPq93BB4lUIrhW9nzyGtVDg+4owqFnW6jox23flAuhNH9W587mx0ScMI+iP9tJeFsdyKMrKCe8iuXIVpmnQons3HWTewxOxBghL7fW1buy7y/CVFo8BK5SJlJjlJrkibgWxGkwwcqhjLstt1wS+5CHprGn1PnwidxT2uO2POIdkqRVaDOSHeoZDkVn/380v37uD6ioex1n3CG9JD5ErZrKQ7U/WTmFz2IF02C5VGhyeO+IxoCbGiKDhSBF+ZpbpaPEdrrY7h8ibc8UkUJA1it5VFdymXhHeP+UPeAUfVLoaFlnJX+/fZrmewZuWveIyaRu2chpcTlSWkVq0nWLyNIfJWTH81pmmimH7uCE1iU85pv/t6ntdPZZEpVO2e+X4lW602BCWt2fZ6KIApKZwySPBzXOh9kwwzdnyQbCJ0zeWhU7J4J84N3MEHbe7hxsQnKUodQq7ajn3JgzFNk9IxT5N45PX/9u/n7/3RZIljlBUMsDbzjjGOgG4Ivh1HHIfULWBTMIuJwTt5IPQPklLS9tufJWtghA76OurbM/v7KdoqaBD6SCKrJEmzCIV0Vlud2e3scUB9AGyOG/KnPddQbQUTldko9cj5AWosNxerMymxErlR/ZSB8vbIMU4pRH8tF2j+WZyvzGKGYwqmaZJcLcrd3k+75gA+Z1jovJZAyS5M0yQQzt7XQyx8+hwmVb1AmncX2a1FhotXt8iWytnpOo97tPcotRKxagoxTZOHv93IVe8to9hK5p6UxzFNk1VmF1a5hrLENRmADWaHP+Q5+mqreM7xAl5cTFHfJX7H9xRYafQNvAG2LeBvgr+mvC4Q008wpBNOLqqtx9Npmia6IRRpHaqEP2Tww/qCmP2maRLSDZ7WzyDH2Nf4Onf8hPXhRHhpOJ9Pe5R3lwrbwXJFbYrR3YQzLRBqbJNG7jVosNVqyzepF0e2uVQZX8jAF2zaGbqjpA6HKkVsbFU68LVNklslr8LLlC/X49aUAz7ONE1MTxrm6dOitvr+fgwdq7ZI/F22C7NVP0x3Kqx4SzznQA1moAZL82DevAMzqZ3IKt/2A9a8R7FSO2MNugDrq2swizfF9J1iS9SrskSdnWFWF4gdgya9vZzjnvvloMaZJn8CdZHAqldL4bnQqQDssrIZIm1mxZ5yKu13L/xe6kb0+B3FwnEVMmL7zUxw8rohAqmWFodUsZs6y8lMW9yns5RPD2kvD2bN51LnY1wVvJbVrstplT+bR07twwvnRtd9LlWOPAOXJvPBJcMO6N4W3jaa57TnuVt7n9MHteau43tw74k9xfdHFhlDshK1iaw3jxOfXW0xliMB01+N9Wx/LCMUUXKWDPG+T9Ye5C3H48w0hjBJ/Q5UN52kqI2dkJzOE04xdvD+GVhfiZI0M1zKuO5jrFn3YY0SmVVWE+uA3/Mz7JuxfP/Ry03u+93vzN/g50Dwx2ruNkBpaSmGYZCVlRWzPSsri82bNzd5TGFhYZPtCwsLI/vD21pqk5mZGbNfVVVSU1MjbRrikUce4f7772+0vaSkJKbs7T8JpmlSVSXIm2U7tVkd/TiOXb8S8hmEsgY0e2yWnb5tmBayZUTKCZ37tpCsONAr96F9fyuB40ZGjkmwS7hSjVKWbS9i/s6qSHma31uHS4G8ojLauMTEVlXnY8De55GwkB7OJFGexqXBG3n85K4EswaQteItKtofRzD9UKwyES3qI+2kriyN4sT6RNcK8al9kOPaUpvQm0xAN2XKmimBjMBfg4JBcVklxcWxEZT0oD/my+ELiOdRXl5BsUcM8sXVAcaqK6neuYKEQC0VtX7iQjp6dSV1xeHFLwQz++EoXouk+6FiFxWrvkb2leHa/g2Vx76EU0nHUl24ynIxa6sJLfsIR+5C9NQuuNoejj+pF0mr3hWZTLYjqaqyggxECUhZWRmKlIEZ9FFZ46W4uJiS8ipkS2fqIfmM2rCBu1buxtV3H75AiCsHphPY3ZqaY59H3zobVm2goLAIy4IntVfYbLXD+eFltJLKKc5cRdHaWaxiL5k9DqWgqJR4OURFu6NhtyCi3qr1bLbctCGSB5yCt3V5o/ZHKatYbg6N2d6ndjEVoT2RbeECsKLCQsbIomzts33JXFhcTGlRPvV12jYqvejc4ByJ7QdSaKUQHxDZJXXJPUgs+jXmnMV+mc8dU/hRP5Qqb4Di4mK+2VDKqXoZj2uvsn5bOsWZp0baVxXlUkIKCVYtPQJvk+xWGdbgvOuTj2JvsAz9owfpZLjw79tDe0lYpp9KR3OP+io77bLCKeq7uKQAxcWCY2eF0Y2VxuGM27OY4raxKcJ/F4THmZrXT0LCxIHJzaErGLP6U7oqeQC4ghVU4KC4uJgUq5pTil+kvLYze6S2GHWByGdQbHZkidmTk9SltCn+mR4lK9mwfRSr9YF4ElN5ovhh9gWuobg4mqUQtBTmGAN5UD+PKxL602fbiyRJXgrrfQ51HY+D1T/EfNZByUV1WSHFxcUUmp0ZIO9gs9KN0eZqrFAqxcVR/qKA30eifa4DfdcPBlKwjsHGKlbpnXBJIbxyPP1Cm8lRythqDaHKF+Lc4J08XudHVRR2XrSB2Qvmc3JxMXJyB97Xx5BlqC1eW3zNbp7UXmFlyXFcVDWNI+vx+xTk5aJoTYsHNERdTRXln1yDe9x9pLeOZpV4a6txo9DT3I5/xza2uXoyxpIoLiyIUfhJNsspJ4Ggr46yonw6AL6qUnLzC1nqa40HP8HSPRQXxzU++UGgr7ST940xHKss43nv7aDBB9JxpDbzjPKD8bwoX80t1bXMN4YyUZnF2trJFBdHSyt8/hBbzdZsShlD2+QeMP97rlc/o8g3gTgzSHlhLj8zhOxkByOKi2ndXqTa/xnvzF+JrNxveVJ7hfneU/jGMZWCvZ2RQ3UETYUe1YswJBmpy2gCiYMoKdl/2dmqxLFU1vlofxDPpSl7piXU7NvE98YQzlQEt4WvuhRl+4+cr8ziX45XOeIAzv2hfhpVcvs/7fMr2TifDGSqTfGuXx28llvVD6nDTS9pD4oERymrOUpZTWGxUKRcO+wdMnI6tnhNN2oie7iwuJjedsZDkZR+QPeRRB3bC3OR3cnUVJaSCOiBOlSifImlZaVsO2Eu7VtlEfBFMzPSpWo2V5ZRXFxMr19v5lbHr2iSQYIu5vxTg/fzjw4ZsPE9DvU/Rz7pLC4qiuFa+S2orSihC9BGKqWNUsqXgUNYoJ7OqNA6lpndmfL5KuKdjZc72/YVR+xRiJYrJboU9hZE3+OioiJKy+qQsPDWVOMLhJi5Zm9kf/i5BgJ+ZjlvpbQ8m5I9I3AULCPQSXA1usuL8TgS0SjgEGkrM2vGsNk9gPSBV6PX+1xGd0nmxk/W4jB9DG7buNSztKKSwW0TePbUrpHz1vl0LAt27RO2zTWHteb5BXkxx5mhIEE7YFhZUYblO7Dln2b4WZ0rbM1y2778s+D2hXDVVVFRXEz80qnEr5xK1ehHSAJ8XU+iuqiArDcF8XpRrS6k3OO7kQYEgkF8nvZ49ixFkVQCKz7E65cxkoVl6FKEzWWaBuXVIjOmsLQixvb3BsS7UFlZecDjTFOoCET7zPLv4lhlKf38r9FGKuELxxS6v92Dswdmcv0RbcmvEOvLYCi6zhrzzAoA6nz+WPu0vIqbk38mmHgI5eOn02pqd+KkABvM9hyrLONM5Wckh4fOtSt4TZ/Hw5LIWi8rK+HI/k7AjPRn2oHwXfuK8IdMgnVVFBfvP4tGAfZY2aRK2yguLuakbmLsKi4uxmcKx1x1dQ21AdF/K1t0SX5tNOUnvYXhziCjYjfBrIF0k/PITRzE2+4LaVdcTCddVJ0MlrdSixvDOoQettPop7gTcGkq5TVRVbSAKwupdQoVZZWktD0MZ+4C9M3fY6kupJQuaKVbKCouRq7Jx3ImYjmaKBs9CHSUixhQNYfi4okx2w92bvo7oqbmwDLs/lSn0X8S7rjjDm688cbI/9XV1bRt25aMjAwSE/dfo/93hGmaSJJERkZG9EVOT0NaOxUrKRkyj2n2WOuEp9i76BPIg+R4d9QJV5oISW1QJn4Mz/UnJyWqpBDv0sCuzNpQJMpxwjXhKUmJJHqqmburjmMHdcKyYH3BSvxZ3XBXiMyPXnI+c80BJCVrULMBc/JKkoO1SD9dh3W2kNu8R3uPlBoXmZlDoxdbvBF5lYjEu/cJA1F1OBs5Duvj6TP70bVdNolx+WjuuEZtpXZDYN3uyP9OhwOoIy4xicxMEQ2SZQmv5eSdJXuYHPKS2qot0rY4rDg3cXZ/VushqMc9hvXuKUiBGqyhl5GybxZWuxFIBMksXQRZ7aHDSKSC+eB2QfVm8OVjJQ1Bzp2P1t3OqovPxDrsZqRvb8SUVFS7zKyozsCpqWSkJGEWB8nIyMDh8pIY5+OYbomwAVabnTlBdmNJMmqfk9FOv4TUbbOwCn7CT1tu/34vndPjuEbK4/zg7Rwmr6O3spcTMjMZLm8kzvSTmZlJbfFOnlRfZEXCvfgsB24pyKE135OZeU+zz7o+mvtMqgmgJGTG7K+cvAGHopCYKMoe3jWO4R/Kj5TvXs2dmngf+sk7qd08hw4L76PESiRDquZZ/TS69Rjc5LmW9buBYeumAKD0Gc/3pR5ur9fuhNK7cEo6W0JZnOH9hszMhyh/fgppWg15ZBGnSTH9rkzsxFzHaM4KfsGUtms459JbogqDNuJyupO78iNGqN+iI1PgTuYHYzDjlOVkUIXf0mjfXiy+28rCUDXtc2T5F/Og9ibyOovAoO9o26F59ad/F8LjTG5I5gdjGBc7ZnGv+g5d5ajRmmhWUZ2aRWZmJrmyg3izhip3Ak84ruTSNh0jz1SSKjhE3oouqbjik9BKdAxXKs84L+cF/0wAclrHljztjEukr7yZwfJWUtKOxhOfCLWx71rmyRdgnHQ+mfUWJ+8knM2AzIFkZGTwgH4CLzmeQz7xaYZ8eSKZhpfMzEcibXXD5MjAU5zXqTeHtTCu/FYYwS4k4MNlkwjnaR3QDruU3KoyUnd9jWHBCrkfOdnR8qpLzj0bgAzL4mdc4E5qccwzgj1gHmTltCHgSWRrZWu6yXncGZrE1Z5kMlMal3Q2hV2lexgSXMKSNV/Qa2BUnWivpmLJDg6zViKX5VKUczSKZLGn2iQzLZEOaXHous4GMwuPJqEpEm470OCQLTyvDcNnjua4pFz0mnlkZo4+uIfYAGer8+hm7qPGcpMgifloqnMSE5p5RrvrVJY7hpKdncPRoSs4XllK247dSEqLti9JTqGbnMeuBIXRg45m6abTuLLkc371unDoBYSKh3FYlZ+E5P5kZg74Xdf/d8K+hGQAlLgU+lTtpjTezQa1A2mZXZHLN3KpOoNFZhbDT3+65Y5smJk9qSooJkWuQ0vvSP6iD8joNw4tPrX5Y5qyZ1rACk8OJyiCU2WL1AlZc+F2avSU93Kk93syM+/bbx9dpX2YwRVkZp58QPd1sGi17BaQwOpyJG8uGUc/eQeP62fzouM5IFbaOvzdHjuu+bK0MBalnYa7dh/9MzMpveRnnnzlNq6qfpbMzE/2e2yF5MDj1MjMzERyxjPLGIRHdXCYIrh7ElPSycyMztWW1YpFRi8OtdVMnaqYI09VFkb6nOybSmbmGWy4b5wo294I45WFVOMhLXUsqnYQvIhNQLIFLGosN7qkUqUk81PC4Xyln8q44GNMX+mgCWEoApIrZswsr7PL7pM95Puizz4lLYMEn4ZDVcnJTCNg7MRSHIzolMruMm+kj9rqchIkH9WSQcbeb5Cq87GG2yTRuxSkhCwo24KMRU3A4O3+z/Nwnz4x15SdWgxUMneXl+MPaaww6NjpIynOR3araOA8wXZ2qR6xVrn+2L4sz/OxeGe0RCs5IS5SMp+TlRmRXt8fOgUdgFjMOx2OFueY343iLKSdunj3HOL6EvUSLEnBefbbZEgSljsFBl0YvY6MY7HWHYkzLgNH295ICx+E+Czi1r2Du8tIyBROpqzkQqAKCxlTFn2rrljbX5JkwCQ5OfmAx5kIFj4jBIL6TWB+p9OZum4FV6hfs6nThVy08y1+MgeyzWyNU9KRMGmXmUw1HnbUBNjtOpfzzcfIzIwt45MUNeb6TLWMft5fwQuZ8SpWXAZSXQnnqj9hXLeesbOfJV4O4Vgn7AhFsljW8w6GHD2RhOQMGsKlySwpEO98xzatmqYiaQJPd3+GO3cX8G2Dd2Gj5xAKqgNc0KZDVIm6HlKWPoF1olATVS/+Fh7JQdGcVIYUWk3tziNAccpAelaswtQ8cPxVVHUfDB+dx1F131KT8zh9E/eCrbvgNOuQ8paRueVdJL0Gc/JK1C8uR8pbhnXIhbBiu3iXPj4Oep+GdeRdja7pYLDS7MIG92AGN7jvg52b/o5wuVz7b8Sf7DRKT09HURSKiopithcVFdGqVasmj2nVqlWL7cO/i4qKyK5nQBcVFTFgwIBIm4becF3XKS8vb/a8TqcTp7NxpFWW5f/YlwAEW37MPTzeARKykfTh0NJ9HXIBc/2HQ95GPA41erxpQNl2pCJREuJRoim/8U414jQqq4vlK3CoMnFOhRmr81m4o4zMBCeZCU6ciRkQlwT7luKSdTY5L0L+0IAhl0K/M2HfCtg6E2n5G0AObkmnc06DL6bqBNUN6V2RKnbDsCuRBp4XIY5sCqcdIko83Jog7W70GXc6kkpvAATfKUl2aqthEmkrS+DDSXF5FW+NncPF6V1BdQqyuXB/50xH+vEeuG4tPN4Rqa402kaSkLZ8L1j+j74fhl4G/kp46wTIGYQ04Fz44S7k9K7Q/xykmkKkIZNA0aj2x+GWTbbJnci30u3nq5Fb4aPzXTMj1yz3DbLeOZAN/o4osiAijwuVIS/6EGbfi9XvbKbsOxm2lbJgWyl3Ob1M0d5lmLyZLXRAlmVSpVqOUNaCLGPqQUJo5GUdydHBJ7hV/ZB40/u7vyP+G3dylFuL6Sc1LXZgHnH2bfDJj+zcuJywEPbxylJ2zy8kwyzmeP1xvtNuZfjgoQwb0J6mEMwZDOvguuBVnJ7cgTnKSO6sf+02+eLN2idgiM9xoib4Vqq0dCTLiLnG3Pj+zEvNRM4ro1dwHW5HY+O3Z+0SjlAEJ4UqmZQX57HW7MQ4ZTkjrNV4JTepqsrL5niulGcAUJ6/k33Lv6Zd0TZBDAz8uvI72nfq0aj/vwMkSSKDSmRMSkmhfz0FoGMCjxF0JHN/h+HIsowuOUgxSkGL4zDWQ2ENcn9BVLnG7MwxygqudT/OeVoI1QhglW/ndusNOoy+g3Xf76Nvg3etJqU3CZKPJ7VXWFM2hJ4XP0VZ7X2k7eedXJNwGB3crQkF/dypTQcgo40w0ncnHEL7esdrkkSOVIrqSf5T5oPUdHtessuCF7lHc/doEaFe8sIs2tWsZa76HbK8vcnjJ9z+Bg5FavHastt05OuBr3B82y7kJubQrTCPWstFkZVMUZWP1mlJTR5X+khfqpN70OlKseAM2rwZS6qTGVbvfJut9uiJx9AnQWVo7msYlT/wmXkYs995jTOVn+l4/09U+E1ODT7Ak2k/oWpZpAf97LPSWZRzISP2vsLV6lcsV8eCqf8hz3mQvJ3dZhYJko81p87j8q/+hSwf3ajd3C3FpJet5NXgFBRlKY9oQq43MSUz5jr82UOpsOLJqlqDLPcERRhaluok4MlCri2kT80GvEb2f7Td0BDh8inZlYhpSRihAC84LuH2tj1wb/0eAElzH/A9Dyr8mGu2PQ7bgPuqaDP7KlZtOZ+Bk55v8bhG9kwzCPhqWKF3RjGGcKyyjKfTH2BgRi8GVwvlzx51yw7oWk9QlrIg1PFP+ywLrFQchEiXaygiyEXqD4JzycZk9V4+1EXJ9cFcw6HXvBn5OzNRY5yyDJ/c44D6CEhOzKCY03XTZEroIsZlH8LIAqF8aSW1bdTPoQ8uhvvE+FGUPLDRflMWc3ucK7r9Vu0jAPzGg8hN2N4Hg4iCrORjm9KFHtWLuJ4NGIoTl03cbjag+0lwqczbWsKL83Yw56YjSHRpBOyxLSfZzbwtpbROdpNX6eOX7WXEOVVURSIlzkmVL0Rd0KBNiodNhTUEdAtJAssWZLGQkAvXguKI2qGqE9K7wO75hOyll1tTGz2rBJewIQTxduPPK2RYuDQlZp/HKSFJUO3XcSgyiqI0kk93aQqyHTRxNDi+JbROjgaGX5w46M8d17qMhewB4pkZIgNH0v3Q/2ykhf+CDocJteKUdvXsexl6ngi1xUibvxJKt52PgqL1yCvfgf4iuJLsETw7Ad2MCNys3FvJhSOjOerhJ9bcOPPUj1u4/IjOTWatMceuVInPIKm0hixPJQRBSrDndtVFICg+Wwc687aU8vgPgmPrHBd0MXc2Ot+esljbui4Q5UKSK/dErljCQklpS05GuuBBtaFLDoZMuL2Zhy3WP0/Y1+B2qAec8ffYuaLCpGH75c7hLNK7cpHStGtBKtogso7vyEPeJtYpxa0O57p190XIcipOeI3M+AByyAuWQcrH48lTckgxa8lO9jA4YXfEaSR1Pgr2LUOuyQd3MlJ6Z7CzmqVVQp1TtgxIbA1pnVtcEx4IPjSORFJ7sGB7GUd0E044y7JYtLOcDh7rP9pfcKDX/afencPh4JBDDmHOnDmRbaZpMmfOHEaMGNHkMSNGjIhpDzBr1qxI+44dO9KqVauYNtXV1SxZsiTSZsSIEVRWVrJixYpIm59++knUJA4b9ofd338cLAsCNRCfBbPvhR/vbr7tSyM45VehAhZDftf7VKES9tF5Yl89UuUkt8bE4B281+f1Rt3pphUZaEtqAmzIr+byIzoj710M+wT3gFsxkLBn9nCNv2bLFX93M19PHkWfLCeKo4GEsSMOdJ8gvgtUC4WyxJwDeiTNEmEPnMjKIVEi5LB8dX2yOlmS+NoYwRazLZ5akYLJKS/A4TfDo+1h4bOCtHrth5Qv+5ii5AGC1NrUwTKFJ16SBIFb2XZo1QdS7XIPMyQU7oI1QiGj7VBB/PbcQMhbSXGr0TzuuIqHUh9mG214xXcLnYp/ZG9ZNL20yhcCI0CfwCrOVOZhWBYhwyS9YL74/AGptpBBUpQcUpOMiOTvXOVQ1j0VSyiuB/0EJQcpNVv5xXkdJyuLUeJ/P6llqyTXfpU5Elr3YL3ZAW+54BK4NCgyA2fEnQHAQGkrXxqH4kto12wfkq028qzjJZKrNvFgXSyJsdSAINUyTUxJY+1Rb2PIDqwGBL2t877nRObjJEhIbtrwjUvJijh+AIyd8yPS3gYyH2njAXhZPpcay82vZk+2fTqFAavvQw5UR46raH1Us/f1d0C6VM1d2nQmKQ/FKMD1kHLRgz7iPeJ7+1LSjWwwO2Bpbo4I/ULq3h8ibR/Tz6bOcrLd2QsjPps9SluoLWCEtZrsLv3pe81Hjc5bnRElO5Q1Bw6HRlpq8xkLYVxY9TLZud8S8NbQRirlXX0sKRnZFF+fy9CrYlXSJEnifccj9Cj98WAfywHB7YnjBv0a7jKFslofNUp8vqDN5XzuOBGrichdGEnxbtzuliNFsixx0ilno8gS7UdNYJVjEFukjrzheAo2zGj2uPTAXjoVRe+7Ir4L3fUPeaFkEBvzo+/nDxVZaIPPQ40XxpQiWZyuLOBlx7Mcpaxm48JvqMjfxYXuBczLmMi2pBGUJvbk7tBFOHzF3G9cxPdZl2NKmghOHAQsy+LL1XkR7jk9FP0ed5BFEMplVHOM8XOjY4tLivG/dy6V5aXEI8oWxiuLmG0MRFFjjV9VU1Ewohw0NkG6pDix4rJQvcW4jBpUT/JBXf/fHbI959ak9CaICkXrOcn7GUl6CXmJ/UUjzdNCD7FQ7MVGgSW+p8VWMnvUji0d0iSCgQC/fvlKo+3bnz2ZKTvO5hBZZDH3dJXjKd+IaS/qlQMgwjZspTWH2Vi2/Y/CNrM1nxhHkOE0OVedC8AEdR7VlodfzZ5kSFXMMA5lq9l6Pz01j3jVoq+8G8UM7L8xcK/7TsoS7OBE5V4Wu64hoXoHnxmj2G1mYaU3ne16R2gSvxi9qXA1Jr/XpdhgyuXB66m2xPsSDNYLMHrLf5Nyot+ZxvXBqwCY3upW1tGFduY+XE4nCrFcHa9pTzJtrMSxvVuxraiWkpoAZbVBdpfWMfLRn5AlyExwsnxPOT1aiezLS95ZjmFaKLJESpwD0xJKVqnxDvwhgxOeW8BpLy0iFBTPuMDRHgrWwvrPosIoQybBif9i/eFTmWGIRXdYzaw+NFtsRWliAR/QDb5ZW4CzAZG1JEm4NYVKXyiSTSRLjZ1GEfn0g1jYZia6+PjyEbx47iDS4n+fc2+/cKcIp0/JVqFmDGK9Mf4loTC81+bC/XVq7HFDLoEj74Q62846zSaO3xPNdjt3WDvapXoI6AZem+T8qzX5Mfb//kTvn/9pO3M376c8b+uPtMn/IRJAdCaK4Gdadnv8OMi1stDQWbo7mgXmsxwsM3s26qrCG2LFHsHXueeXj1DstRIg5p6bhe0eFoghpSPsW0ZNG0FlkKy2PM7VX9sdTImoJElNtjcaemYBzvkQXMnR/zW3UJv++XEAvFIcQ+WoII3icIu1UNuhsFbYe7Xu1uwws8lOciHHZWJYEsvO3yY+85QOEKyDsCBPn9PE77ANEfJB7lKYE1VD/q24TPmWvLy9XDAt+jkEdJPzpy3DFzowTqD/dPzpLrEbb7yR1157jbfffptNmzZx5ZVXUldXx0UXXQTA+eefzx133BFpf9111zFz5kyeeuopNm/ezH333cfy5cuZPFmQX0mSxPXXX89DDz3EV199xbp16zj//PPJyclh/PjxAPTs2ZNjjz2WSy+9lKVLl7Jw4UImT57M2WefTU7OgTkT/isRrBXOit7jxf8bvmy+bdl2UmrEFzlmMR+oFj8AI68jLStqzCR7HCwye1ORHJtuC6IO3tkg9THZo8Flc0WWUOcxDO/WGl9YelG1HUPx0WyTvm2SkBNzopNJGOE61WpbxcJbCtMn0BQyEpyRSRXAWU8+tSGKqgP0yk5k6Z1juOYooZoVauA0+socyW6rFWevOMfOwtohZED9lbDgafjlXwCkzr2NV4O2MllWH+h+PJzwNPViGzDnAZhtRyuOe0IoQAAUrhP7hl0J5TthxZvE7/yOQmdHTg1+Q3LZaoqULFJqt0WcWv3bJPHrHWOg+wlsc/amrVTMxW8toy5giAkm/Mx2/8JkdUbknm4KXhH5e6Y2hr41Qop8sykys4KWxg65I+lV6yPtsrv/sQoFzSErOZ4aOZEJ3ulsNNtz1dnjMSyJz/XDmBK6gOO01VwXmozarnnHcKI3qnTiIEgXc1fMfskMMU0Xn1OJlYi3rgaX5cfhTuCd1OvYm3lkTPv0qnV0NnZxkvIrbr2appDcSmQ97TUzKLaS0cv3RPblkcGXcWcC8JA0lW+M4TwUmshrCVejWzKtK1cQsDSKrWRqtAOUrf034fjgPwlaCkfqC3Ggc07wLma0vY0ntKn84ryeFL9whBS4u3JZ8Hq2dzwXMz4btVY4AfWgn6e1l4mTAjxRcyu1rUfxgusKjFCg0aKjPjIrV0b+VtT9KwSFkW6WovlK8PuEo+CE298X/SUn4nY2fT75AOTEfyvmqSPpWz2f+aM/Zex5t0a2a5qK5K8kIB1Y+vCBILtdFwbeOZeed/7Cr3FHEapoXp1vtdmZe0IXRjdU7OHcxLW86f4XwR3zI5uvLHmYLlWLUVLaNuojYGlUb19EcN9K/iHNZEjdPFpXLKPWcjFc3szQvLf5OWk8CWNvJaAmEOTAP0eAgkovns/+QXGhKIkMGgZfGCNj2qiaMyIXXx91pfs4Ql5LbVDHtDMASq1EntFPb9TWU7qeRMlHYmuxqN6ZcxK/mj0JOZPBk0rIsHCbtTjims7a+k9FXdsj6OCfTmWrkdzEjWR9exGTQ2+RYFSxI2MsPxv9kJwH7jSSFAemJZGACHJUWXFYB0jEWR8rf3yX4atujTk2EPDR2y/GhAypmnf1sQwP/ELXwm/Zmn0yt4QuQ7b27zTye4VS6+9j22kZhyvryJHKSG0VDXSELIVEycsSsycnG7O5PjSZY4K/XcHN5RaOD+kAHbGFrk7UWsIO00PC4TG6aBrfGsPpIBcRH2yas6qvtItRygY67P0MgN1mFrONgU22TZeqCWBnb9dTz+Txjqx985oDus76CEhO5jhGw31V1CT3oEx3ostOpLSuhIi1OzOkKo7KDpKR4IyoaAV1kx0l4vO2EAHQkGHRuZ5qlG6aKJJEnG0Pr8urIi3OgT9ksrO0jo0FVYRkN8/p43klc0pUqupffaAqD/YshuLNBCUnmyzxeTdVDqTZSr31ZdnD+GxFHqtzK3E24WzyOBQqvaGIQ0lpkGnkVOXItoZZSPvD0I6pnNCveeXJPwy5S+Ct42HxC3DE7XD5AvBXCdXgpDZQtk20u/Cbpo9Psh2WW+0gRz2V6C6Z8bx2/mACIZO6eqppDRXUmoNpO0SaDDLXx9JXcPjL8KuJkNyejkMEEbTfnYUPF7VXLOfEnrHq1SvMrihmlDs3Nc7BN9eMom/rpIhal//HB+hb8jXlibZzSXWBJPGmPo5Vpl3G2H8ClG4lft98QpZCnqflzPT9BWoPFk06jbofBx0Phyxb9U2113h2kKHOF3VmBywNqf76rt9ZcMabbI0bTGe5gMxEJ5z1Dr0D01gz7Vqszy6Bit1CUc8uOaT3qfb6sd61GIGo4t7vQBc5n3MVkbASVr0Lvw8O9c+cKf4++NOdRhMmTODJJ59kypQpDBgwgNWrVzNz5swIkfXevXspKIiqEBx66KFMnz6dV199lf79+/Ppp58yY8YM+tSr+7311lu55ppruOyyyxgyZAi1tbXMnDkzpibv/fffp0ePHowZM4bjjz+eUaNG8eqrUdnS/5fw24vaQRfA0Q9C66Yn9IaIGVg2fB79u8dJuOKii6h4p8Iu13lc88twIFrSlUU57srt9PFHF3dge7lLtoosodNf5+wzzyE+3p6kWx8ifnsaLJTP+xTaDY/dFnaAHHkXXPqT3XnsoBzGl1ePZNldUeU9t6bga2YSKKsN0L1VApmJLjqmxzG0Y2qM00iS4BxlDhepMzEkVXj+f31JKD2kdoIxU2L6u8f7qPC4z75XDJzJ7cRgmpAtOgv5wZMKU8qh/YjoIOhKEgNs+2h2nrt4FW94r2V89ft46vbRyirGaQnja3D7FL6cPIpWSS5IaEWeuxsODPwhk/K6IKqlR6PCPU6IRimAb5Uo2fINodeZZQwCYL0losDlKX14OOFuErvYn8F9VXQbNLrJ5/dnYLcsjK17QxfQJiudbVYbnqy7gyvVrzFljd2PnsCors1nPvUdMISFA0SEQ3W4UImNan6YeT2LTFH8NiL0MuUhjR8ZhpKUQ5UjC58cuzCSQ3WYDkEEuM/ZuclzZrXuyObTZjHXHECmVIlZHR3vEo1KLg+KtP/BbGC7lcMEZR5xeYv4SDuZRKOCG63rWWp2p93Oj7AOMgPjr8RYeSUOyeBh6znmmf05XVnASRfdgc92drgTxHfy1vJ7uVb9Ao8CeFLQQmJc2rd1FX0l4cRTzSBJwUKO9n6DuR+nkbs26giUD8JpZMoOem15kfI3z8FvaaTGt+yUmTvwGfqOOfeA+z9YfGbdyNOOqQwdNpL4pOi4N7T4E24OTiUoH/ii/EDhcajgScXyVTTb5rTg/XxZzwHjLF7L2cHPyLDKserJGmfr+3BKOoGsgRwXeIQnQmcRsMTn9mbq9SxwHUmgooBaLY2e3hW0qV1D1u4vuUL9GtkI8HbdFaRVreH7NteyJGdio+toCWZdGUcrK6grEan5AUPmKf1MNptteVcfy0v6yagOpxj7GqCuvACPFCCxcCmGLTSQLlUzRXu3UdsO/UaycMSrtO8sjHFfUmfODt7Dlg7nsa/T2byQfCsfS8dFM0b/S+C2vLyhPUFW3WZWKP0j2x2eBLpXLmC2OYjCjqe20EMsJFVDliziJbFQypQqyHc1PX62BF9pLj8ah0TKigBqKkS2wTyjPx9IxzM36TRwxCGH6lBrC6ixPMx3jt5v336fcCLManXpQV/XgaKMJLqPvx2Xy83HSSKQWiBnse+yTVynfk6covOa9hT3qm//5nPIikzAUvkhbv9cSACP1txJ+q6vgGi2lar7uE39EAAXwSaP63HGvSxzDEELVAIwOvgvrgyJ0rp3k6+KafuwNg0JiznGQBrG7MrKyzhYqEVr+IbrADi96FmuD03DUFxYF88ku1NfJg4TdsPQNm4GyDsgUI1TVaizM05ChikysxFOwnCAM2zDghA+UeRohoVpQWqcWADH42WD82Kyp3Znuj6G4TU/wEThPKOuGGoK4c1jYfssBv18EWcqIuOxqUyj7CQxD9UGGo9VFV7x7BvyJgKU1gaZt6U4EhQNZyrdebwYqxz1nEZ/W4QrC1a+DQ9lQMlmWPIyVO2DrF6CrgJiM1fqo1U/yO4Pm78GZxJkxGbvOFUZv27graek19RzDmes1kc4IOu3nQXFNX42FUQDhfviekf+TvDto8aRCdetgbhMuHY1in1vmds/4dFdp9NGijpfHbJFZytaVuYN6rg0hWSPRoU3iGVZ7LGy2Gi1Z9XwZ+2DhN35kH4el4dsPt5yoRQpYXFo4Dn2pTZYKzVAeG33xBn9Wmx3oDCaeG6AyAZyJ4u/HTZ3YtthMGIyrd3inc63UjkveAeqUu874UqCPqexOm4E94XOF++96sBvJxdI6z4RFRtj74PT7IzTDTPgsJvE3wMmgisRrvhFZKz9AUi0s5GrbXXFsHPXofzp7pS/Bf6Su5w8eTJ79uwhEAiwZMmSmBKxefPm8dZbb8W0P/PMM9myZQuBQID169dz/PHHx+yXJIkHHniAwsJC/H4/s2fPplu32JTZ1NRUpk+fTk1NDVVVVUybNi3qkPj/ioRWcONm2PwtLH4Ruh3XQuPo5OKpHw0xdFEvHJcBb4wVaX82HA1SZjMSxBf7O+cdnLv8DK7ce1PMfrdDgc8vEf+8eTyUbke6cjFc+J2obQbhTDnlRfG3acDaT0QKc33IsjDScwZBZi+4akl0AGmAnGR3jCHgdsgRxQyAuZuLI95yX8iIcZg5FJmQHh0UZUmivVTMIfJWgrI92cmqKD+7dpXwkgP0Gh+9AH+l+L36ffjgHBg4EU57DbqNE/cRqIFCQTiJ6hSTXlwGVOfBd7dEujENPeLsCQZ9dNe34vYJZcD0+inE8x5ldOUXMY4hHHHC+3/dWuh+AppdYphIHbPla5iuH8l2M4cOVi6Xhm6ixEriHV3wgCjlOznCWEyvwaPhvqomn/GfiTWWyPi6SfsER2ImpwXvZ6i8mWypHOUAskAkSWLk+Mux7ipC0Zy4LT/7HuqHr04sELRQDVPUd1hh9eQ05zLqKgq4N3g+zrT2XFXyIF33xJZHKboXHPHsMLPZ42hMWhlGj35DuUCdxcf6ESw1uke2f28OpbO5G4Cg5OAe7X3+oc7mefMhPGNuo2/gDY49/WLaJCgM2vAIi+87/GAf2V+Ge+1FtkMy+NgYzXHyEiFRbEf142xCc0vWuFL9mvYViwgkdWK7IhaLlQU72aV25PbQJVRJCcT7Czkn8CkVno58E9f8grTOlcluM4vO/ncpSmyc5dgcTNmBA53cYDw3yrfst/2Rp1yE63dyb7SEsLSsyx3rHJJUJwvNPryQduefcl7LkRBTBtkQD6pvMtN5W+R/M1iHLrvQZSdmsC6yXbMCqE4PQzqm8fjVExl35RPkqmKxZrQeykZ/KmZNIX5XBqYtuW4FRRRVMoIkWDU4HG5aB3aQUisiynmVPspq919WE6oWJWi1pSJjKlRTzFPaVKbqJzFeWchV6lco8Zm8YjQmNPZXinGzb9EXPJMo3oPdZhbZlDdq63Q4GTkumsXau+Az5jluIDVUQLJVSfvaVbxlnYCc3HyJ7H8i4mv3MOroV5YAAJ6fSURBVEZZRaJexjQrKnrgSkglM7Cbs5R5xMkH7tAuyhnLZ8ZhTNePJKAbJEle3KHmHZdNoTR3C9mFszlGWYG3Vry/Ad3giQXlHOJ/mdHKGjp27cW0WyZiOeJRQl7a53/LCepSfnA2LwIShh8nT4fOwNpv0cpvh4KBqglH9zGXP86ZgSmsVfvSxs6IN1K7cbSygovUH1rqZr+YYYyiunb/ykgAuuLCtL+X4Swg1fCRIIltmtPd5HGDBgwAdyrYyruPqa+yzXU+AO2DW2PabjLbcmvoMi7RbyHoiGYWfG6MokBumne0ORgBL7qvJppFqDr51ezJgrQzYcMXPDEmkcsOE0GvUZm2wyvoJdkoI2Q7GwN6PaeRJEWcOW5NYe194l157qdtqHbp2KRRor+0OPHZJVFHnCTGqXnOGzml4m1BLxDGZjszpkaMNSlSTaT/hjjzkLZcN6ZrjGMjjPA2tRnnz48bi6LlaXabCUPaRe6xueP+NmhY4vrtzeJ3z5PFmqD/BOhyNCjNBJFaD4Kjpog1SqAKesY6Sl2aQsiwqPVHn623iUwjw2y0iYDt3QzYQeZzX1vCcc8uiOx/uEKsWSzNQ4m7IxWejmL9oqiQ2pGALo5LXfQwAL2k3Zyv/MC96tsMlTaSbFXbsu0W/pCJx6GQ4nGwuaCGgG5yjLKCB7S36Zw3Q2TS2BQcS5xXc4Mq1BJJ7gC37Rb3gBKz1mkK4ffvQEnR94e+rZvJsHUmwu4FYs1gl68jyRDy0WuY4G7MkcoZJm+KdRrZCCZ15i3j2Jhtddjj0JBLxNpWD7BoeymBLbNhiV2+uPp9qC6Aki2iQsay4OFssPmODhSmaTF9iXDqhb9B1fZ44Q8ZaIr093fI/kH4/+Ea+x8EQj7AEvw5tYUi06UpbJsFRoDaXuciYcam0JohqC2BycshrQsE69j96AnsfvSESFptGOnxYkJ9Q486pxRbtvVh9Q06bKnHfVSyCfYtE1lHbQaLgTaMvmfBpNmABN/cALWxROmA8LB/eRU83AryljebadQQyW4HVT5hSAR0g4veWsZnK0TmQlA3Y2rHNUWK4TRSZAmf5SARLzVaeCBUhNPorRPh7ZPEBDdmCqscg8X+nvaiRVbFhLLibZF+2/oQcWzJFnjPLovQA+K5hDOOMnvB6W9AWldMPYQiWRgoGAHBuVA99HoA2qXVm3iNAEbO4JhSDW/vc+DEfwnH4d5F7DVFCaCTIG2kUu7RL+aO0CU4rABvaY+TIVVxqx1ljC9bwyn+rw7o2f4ZOMpaBMBweRMuVeJyVRhjO85fTr8rph1wP5LmgqS2TNEvpI2+h7wqEfE+u/gZ2sklzNGO4Hqmo+ev45/yVNySH1NWscxYQ+5X9+GUZwyls1xAN8f+I6QSFsuqk7k851Pe7/oMFykzCSni89KlaJZMjeWmU6qDdc5JtKIUQ3bgkHS2/A5uiz8bQUvh7tBF7LZacZP2acSIDjss4+KTAdDt0lPV6aG2zRG85zpHHF9bhk9N5DxlNpoVRHN5cBCkwt2eX+PHNj5hGIZBB7mIj688jFFdD1zZZVHmBOpw0VYqiX4//45QXTgJYrkObEw7WGzoNImP0q5qcl/QV8dEdQ45UtSBYga96IrLXlxG+V40M4jqcCNJEn3bJNGvTTK/Og/lK2MEffQN3Jx/E/vIojhtCMgqkqljhsTxATRcVgCHO45Dyr+jT7H4Xl/x2Otc9er+F8whv3BeBatF9DbkrWWYvJlnHC+xzWrNFrMNanwaz+qnNkqhz9XEAnC12ZkShyhv+Ng4guXW/pUKHbJFB7mItpXLyarbwqXVL/Ks+U88gcL9HvufhMRMUWKrJWZhoHB98Co+0Q8nMysHHPH0lXeTVbPxgPuTnPGELIVz1bn4q8W71afypwM72AiCaVD19kR6BESZdKBa8IysXPwT/dbcTy95D/vOnUe/kwStQSC+LcVyOpLup6ecy6XVL+z3ND7JzQarPZM3nYcV8u+3/W+FrIqFnYbBcHkjbyaIzKbci1Yz/JKnWSn1Zpncv6Uu9osul77FpAsvOaC2uuyOfK+rU/twcfBmqokjk0pxvc0t2BEBAYwgeijIBHVeZPtx1bGqbdlSOacr87lR+wyjNsr3l04VW+UDzzjzBQ38j3Skww8XYoSzUVUP+VYa+YkDkebch1K5S1AhAHKY16lwHRcsHhfpJ6ibkSxyCXDZmTweh0KiS2Nox1SbE0jYg0M6CC6ucL+nKIsifbmkEIYepGiamNdI6yIcSK36UdvxOJb1ncI02yZuSNkAwtnTp3UStYHGzgxfUFxjUwvrkV1EdqqzQaZR2HFQUOn7+y9stXoOyWMehisXwsU/iAz85HYw6gZRbdAc/05iDnQdK5yXAF1i1cjCz6bCG6RVosjoKq8LNnLQBZrwGoWdPuFypHA5YxgPaG8BUBmU+Vw7mR1pR8TsDz97yRLHp0nVZEkVZNtzq4pByLDw2+dxawrJcRofLc/lvV+jlAatd34s1kn2mJQuVTNZtalGZBlsh28Qdb+fd3ht5/mDytTuPak3mx44tvGOw2+GOxuUh429D8b9E1r1RXeJz+sC9YcmHZt3HN+DWTdEA6b3nNiL7Aw7G3vpq/DhufDj3Zz7+hKq964RFCXt7MqMks3w2SQYdoWgZwl5oXjTQd3XnnIvd36xjmn6sWy0xFxYXhdEN0wCDdaJ/+34/3On/4MgUv7g7OjA/N5pTbdbI+TMtVOfw0KOpOEBwoFRtA7m/lN4ivWoMdXQaRQelIdnWVjx2Uzt9AKG/cpNVOfQdvmjomFnm+DXDMFrQvUgBqojWq4W8sZOLGHcVwXtD41e4wEi2aNRUSfur8aOPizaIYyYgG7GZE9pityA0wh8ONhu5fBCT7uUIS5NREF2LxCD9zkfwJdXczvXiv09T4b0bsJBJsmihvujibB8Ggy7HEZcDYrtPGjVRziS0rtCr1OEN73vGXD84+xJGY6Mxeqs01gVEs4/Z1wyAG1S3FBXJj4jPYDSbiiXnR3l5/CUb4CNX8LSVyCzF7fqgnzXKYXQLZlvHHfxkeNBHFaQ0coaAEYpQkbukBW30TV4cAPuH4kMTRh9G832OBSZ61RRLtm5U1cSDjKTUPMkstwUWT9W+W4AZJsg9VZ9KgHJTbCqiLPUn3EpYElKI4LeRdpwvBkDKblyI2Mn3tzi+dZMWEpW286cb3xOqluhW0KA7vI+DFWkGX8RH81gKFfSSNUMEiQfqYpPGONw0BHn0j1/zWdlmSYOyWCAvINVcm/6E1X5GhB4lc7+d1Hs8cG071d1xZMe2MclhQ/AfUnsdvVkftoEuku5fKiNR3N6cFpBMvJmcUHNa82eu3XvQ3lFP4FD2qcclFFckdCdNKmGnvJeXtf/nCyeg0HQUghYjccuWXMxStnAhRXP/innTXCp4I06PItr/HS+8zssy6KsIMr5ZdqlKtVqGnvdvZiROondKVFn9BTlGoyMXjF9fxp3DteGrkH1JOG0fPyoHUlRx1PZl3QIO1y9sEJ+PjdG8WbK9bikEA53PMgq/vxNbNi5j6+dd3NhbWNhhYYI2bxUhQki00wPRqP8CfjIlspwmF7e0R4hFIjNuNhJa64NXs1QeTNTKsR7cKqykD7y7v2eV1bFHCepTjRXHC7Tx2Gsxin/edkp/w5k5bSj8Lp8eg8ZjZMQt2gfcdTtnyJJEkqCKAd2J2ftp5cockoWcLbtVPDX2hlGB1h6u+GxMcx+558Ydsy3znIStPmHlOINTFTn8K7jUdp0G4jHLoktaXc8VxWdjMNbhEcK0je0br/nsYo3c772E0FTYffa+fttf6DYWVLLB3OW4Pf7GGG8gZEuSmg0ReYm7VOODs0DoG37jmiaRvfb59P79gN0qDWDQ9qn0CXzwOZHU3Fi2U4jo66cAiuNu5y34ZXczO3zCJk5zWfRLcw6l9XpJxEMxJKH7yWWDydZquMEZSnXyJ/R/g3BdVJRF6SLLBQdDxQP338TcfhxE8AME2g7EzhbncexpW8K57RlRBTJAj6vyNzO6h3TT8gw0es5k8OL6XCm+ZAOKRRX+yML2kPap3DxyI70yhHZzW4pNhvSRVBka138A3Q7FvJWQE0hfaZVccn6vlSSEHOehkh0qZFMhvoI+0rCZNn1cWwf8YzDNutdJ/Tk8dNF2VHXzHhGdc1gTM8sBrRNbvKcfwskZIsMeFcyDDgXkttG6Sgq98ILQ2D+ky12AUQD4l9eHbM5zAVlWvDGhSJQdM5rvzL4odkUVkXXMuXexs8+XIa0vbiWkGHSsBIrQxKZ96/pJ3Bs2VskSLGO5o7p9vfPrjbwWU66S7kcqywDoMhKQTdN9lX4cGsKiW6NVFvxbWdpHc/r4wEIptkld1Yz46W9Pgrg2C8f2x+daaTIUtM8SZIUKaeLQHUIwSPNhXqoCFqtNzs26TTyOFS6ZiVE/p80qiPeXmczQ7MTEir2RIiwH09/GC7+EZJtBeWVb4sqlHGPEPnQmlpDtoAw79Vmqy2fGcJ5dcbUxUx49VcCIbPJctH/VvzPafT/CbVFQjkt/IXRW46eOV8ehgc/pfXLA4ZfCTkDhcNBD8T0kZ3kYlTgWX45SRhY3WzliX6h1UipHfGq/8feXcfHUacPHP+MrGTjnnrq7tSoUCpYcXcoxV0PPexwuIPD4fCDoj+g2BUpFCmFllKnQl3TpknjydrM74/vSraRJqVpKs/79eqrye7s7Gwy2Z155pFU4qlC22GaBStDB0RBn/pn7FAC4q+Ce1NVxNgOqtdQm3CwpRFBo1SPk5mrCli4oTgSNFq/vZKtpVU13gwcpr5DTyONH6y+fG/1pUXxPP77y1r8B18HQ0MfVIFKFbhZN5NTKt5hvtUBe+F7avvK81HXtEJvkOXbVO1zXEp0CkBpnnq9pltNRVgzQ421XfwRf6YM58ncp5nZ8VoWB1T2Sdq3qrRiYLtUlaH0/cOqx9LcNzl289ORSSCJKz6FZWpMMkXrOE7/CQAXfrw46K6vQ9ds7rUvYpnRmWnJKrgYCKWq+9g9HzC7ouM1n/N84BgK7EQ0XfVqqJ7J1hiuqm1851Ilk/7tKvW0+lQdr+4hUKZOph1ON7Zm1Mg0On/bv2hd+juZ2a1wOOrvp9O3e1cc8amMN37nuKLX0V3qIGJRlppQNzdOBT1/DvZgjaMTyRlqP49LyearnEv4PdTsMBho2HSZ4pISMl4dSv6WjQ1a/q8IWjYzgz04Uv+Vz7VRbCadIlsdJLTXNjNUj2YhzGx3KVvtFMy4eJKChRxj/ALAaq0NGzNHUkocQTTMhAymWoNxVWwmJ1h3E8Mu7dtxyX2TG73NY/PUhLSNdjp7Qw/DCb4HGeF7ssbtG9qfwtOB47DM3d/TCKBj0Qwu3npf5PvtaxfxrXkNeYVFbM9by1bUVcDiApU9szR5FNNzJpKf2J0CI9o/bGagG67E2Kl1p1S8wxr3mbjdLkzLz4gtb9LRWs2azEP53TOC2Vmn8J/ABFqUzmeM9zGcyS2wdYPRxnxW/qaamXbWNrAzpY50HvKfzlaXOlD0+6ooDzXzbattIUmrxKGrxsPeqtgT2n4rnuXM+N9wa34VGAacZ7+N+9wPdvq8hjM0it7pxulJIsEuw9BsHI1oCr2vyEmNV1OaqKKVVhCZpGSFAoUJaQ0PGhmmg6VWG6psB+XlZfw3MI4KvfagxtqC8kjTUYBUfx6f5SXxSOq9vBEYT7zmxRvqP2TtWLoekuFdxxr3mQzYNoV0qxCzAY2wA+UFdNE2UG4kUVqye0qxl2wu4Y7Hn+GMHw9j9hu30c9egsMOZWQ61Of+KO/0mMfEu0zVe2wP+az1dczKVqX1rk2zeML1Av0Kp3KV70raDjul3seWxOdS4MjB7409ttwUiC1buc9/Fp8FYwdWPPrALbTUCrmq/KkGb+sV5pRIoP33UOn6pg6n8ElwGG68oczv6L6z3aerY6+gl/z0QYSb5foCFsFQqZqmqbYFED2pTo93UVIViFyUyEx0cecxPXCZBqaukUAl6yyVbV5se1hk5ZJKqQp+ZPdUF/xCx93F1YJB7joyFNITnGwsqqyRARM+/tzxAi1Es2jCx6xt0jycOkgNJfj6+kMY3yOb9hnxfHzF8BqP3WtoOvz2Mlz6o8ouqs62YNvyaOlRfZJDwxi2LI652RUoIw61b+YkuSPlVBW+IEMfnBbpZXTKa4tZW1Ae89hwptHH8zZxVLWytLANdgbX+S7jzeBYhnhn4thh8uZ14zvz+dUjIE1l0n1sjaBNqK9Rhe3iY2sECzcUM23JVt5zP4Dx4ihaBVS59eRf1zHRUCPqtYzO6iL5jkGYME8a9l1F+BtwnB4+5+mSlbiTJZvYyBv50TGc36yumA2c7OdISOP20pP5MniQGvKkq/0+z9EG2g5RLUpSc2Hdr2rw0Etjo43UzcYNFQn3vb3DVMNSklCfN3PWbqcqEKy1Mf3+6sB5pQLKtqppZDm9o7fV2rgsdAZVuJKnT+7GNWNDafq/PAefXgt5oUygU9+IZgmhor9vThpEq0S1W4WDFOXxbWHdz1y//Cz66yu43qx2MD76NlVK1uM4SGoFQX80+BMWTof2lkLXCXVHicPLNSZoFK8ec8zTP1Eayqj6Y1MJg++fRkmVPybt0GnokRp4UOm/S+22BGyDsZte4O8fL2LFwl9URheo3kszVHbAheYXTLf6sdVKxnYnQ/dj+Zt1OcWRLC4NfnwMPr8BkkJX5Xyh9NeSjTDjCRgV6gn1+xv0Wv0ablNn5PrnGaPP5a2Ma3A4HHx57Sh6tkwOBaVCDpoIZXmRVGjD9kc/kPOXcn/aF3xw6TA22hmc44tOMvzcPxADi8R2qmH60l9Vlst/ghMa/PPd3ZISEujRuSN2KPNsnt2JpXbNaU0N4XRG9zNfWREAQXQW2h34LX40K909KQk6Q8u6+LDFDcxtcUbMOrr4/iC+jsagtdFDzQADrnScnmTWWlnkp6srXmcXPcciK5fz/LfwcvbtJCWlsOrSNbRsnYvXncYN/ssA8FaV17X6GH7Nidd2UFa0becL/0UBW+MM/x3Ea15OCEzlJ2MwU4IqCPaZ6w7ecj4YWdbl9nCE9yHsloNweKInEwetfp7RxR+TppVxje9lnCnZ3O6fSN+lT9QYm7w7mJpa59TgYLxmHQdge9Bt55/A61fVbFabYJcxSF+G1YiR5o3h8KTgClbbp6qKaKdvpWDJj1TaDhaZvegTeJ3tWgoA2fkz6Fv1K6flP0mflS/y25t/p6qilJ85n/iK2ODeQK+6iup0x2PaPkZVfENqsIBuJT9xcOFH+H1eRrlXcFrxKxTYScS5nJS7VPDB3qAeW+HYeVleYVw7ErRKOqxTZTBl7hbca6opTD8PepL8SxfhdKvPjYAvNmiUUboUpzMUANHUZ0e7Ln1p0zE2a6o25W0OZZ7VETwZeFKy2GKnhF7v/hc0CnvDeTpVdrREKeBKo8BOJCW94b1odNNFF20Dbs1PRWkR6VoJPq32oPvlj73GG1/+Evm+tbaN0davmA6Tkbrq/1dO6OddVcScxDGsOO7TmHUk5P0a+frF4ASMuq7SVxPwVuDTXWwxsnd6ArZ4UzHLt5TudJ3ff/Euk50PAPBr0uG8ZdyDs0q9P2uhkyXTavjnSVNwmzpWKBsvUFVG0BHP3xzvU2k7ccbV/z45btMLjF3zL7xBmxnBnrwYmECF7aKVJzZI11nbSBstP9Io31dVwQMOFcQ3amlWX5cWWmEkW3igpk4IE+ITSKZc9X1pOwwr1AD38dP6Mumko9Ux54bfmHfQw4SPdX1BC38o06hlSlykPC2cNREuQ6stCyLVqKCttpWPrBEsGPwoE31/47ngsfxqdYfMripjpv+5MSf54eBTXZlG4Qbbg+77Jub2cF+d2sphwrft8w15Z/xbZWbtKHyirzUgqyP8971DKaXxSDtedqhMJZfDIDcjdn+u3n5i9bbYjNTqU5b/3BpbmgaqvGyNncPjjucA0J2x5yku01DH5uPuBuDiLmVMs9SgmXeDoxmrz+G0F3/h4alL6e2fD3kLaL31u5j1L7NaE9cldoLv4/6TeDKUhRQWbtau1VXGFxIusUv21N/7qMlpGvH+AvwYJLgbdv7WcftPLHZPIksrUgkMtZ0XDjgPOo2BFn1Ur9jybeCIh8GNG24QbpaepFUw1XULC9wX85rjYR4wX8LrtyLvFweCffzdRTSKrxw8GarU6ybVZb/WbKNqE8vGdE5RU7gAFrwHc15VZWQA6R1jIramoZP75sG0nzwCgC6hdMIWW6ZHlsmgmIuHVktVjkuByu3Q5zTVnNm2VdpidbqhPigSs+GMejIKDn8QrvtDpQM3UIqnWh+ZUNQ9HFUuLPfV6Gn06JfLeOH7lZR7A+SVVDFCX8gTzmcpCqj1ZGz4WjVZ63wYnPJ6zIfWNeaHlPz5M5VB1fD6u0Xrma4PgeS2oaQjHVoPhkmhcaGdxsEt66NBtPA4USCtbDlXrbyIjvnTaKdtUSOGfRV0DQXqKKsWNHInQ8DLZXl34carsmnCYy+ze5FgBDkoN40qnPxerY/HT47LedR5OeXtxvBk4HgKlv/ML44hHHrJ4w3++TaFUefdw6hLVUbGxlZH0Tdt1yaKmc7ovhso387CDx5kQ6WDPD2bVLuIKVmX8ZtnBE8ETkY3DNyaHy0Qe8LptKtwxDW8LM4ONS9MHXI6Zno72ulbOWj7FwAkBIr42erJn+5zGWipk6EOOeqE+ci857nJVE24vZUNbGhashWX5sdb2vRBo6CviiuMj/HaJm0dxVT1OpOPg+p9IM+OPenvv+1TfnJdQ7zux+WJXuFKqVhDoqVOvApJIcmp85HzTtx4KS1v2GtujHDJ3yTzf/iN5g8aHdo1K1LuUF1W0TyG6EvVwU4TcMYn47GiQaNwqZe/ooTC1L48mvg3Opn5eEO3tyv8iS7lv2PqGrq/lINWPMlvc2aRpFXgcsVewbM09f5ZlT2Q65x34bIqccQlke7dSMfKBYxY/zwn6D/SK7iE+e6LcZk6C9PU+/dxJeq93m7ASULCxhlcaU7BXaGyofIqTeYmjOLHYC8CmT3JzGmD0xEaI75DFkRm1Wq8WaqMw9Ibd/BsxKdwuu8OKlsOJbtNJ470PcRN/otxefbfoRt2fAalRA/Q27Vtx0DvC8TFNTzlX3e40DWbuVYniiw3Rxmz+DL9nFqX/dx1G53XqZ564emRo3w/cmnRP2mvb+GJwIls96hylOWuXqxqMYFO/WMHBniyon1yWmnbeMbY+RTEoLcSv+biweS72JoRnUK0YEMRW0ti96Efn72Cb5+6dKfr7GWpcuFPg0NJdNjomo1hxmZWb9PSanvoHnPItrc5eL3K5rCqSvEbcZQYqbzn+gduX/19+3RdRw968ZpJnOW/ndcDh/HFhF845NpXY5Zr028MJR2O5pHAaUwOjGH9ulWR+0q1xv3tLLZz1Tp1dczTqmwhhxgLwOHGPubfBLLURdIT+remvb5FTdZa+hnjvxwTyRjwBy2ClsWIThm8fN6gGr1ewuVttZU/P6C/wCh9AdeYH9Fn1k3MszsxUl/EFf6r+W55Ptb632DeW1jO6GddQqgcqK6gUbgPUfkOTZp9QYueLZM4Z1i7Go+JBI325f4q4SCHq5bMl8i5RgNKf00XXPRdzaoFICfUQ8hp6PTc4fO2+kXhFTv0LPIGrBrT7qoHEa/0XcWfdisONtTfuMtRR/AjlBFzevy8yICZTXY6pxjREtiFKaoXk88VzeS10HkveAhat6NjVvfv4En8K3BqrU+1swTqraU7HzKxp6yc8D7uQ65rcIsBR5zaR36xujN7wlSuKFTtHWLyIEZerwYpdTlc7QvxmXDjcnWu1Qi1lYqONubTWstn9bYyyTQS+6nD74fx96iv41JUvXVtB8lHPhz9OljtTWXT7+r/7seqcqkPL4HfYg8GwpbfdyStU0NX/6r9FWdoxbh6Ha3qTU94AdoMVncs+RRWfQ935KmUwh0ZTti8oM7nA1QD6vJ8lcbaQKnVgkZnvfRrzH3bK3YMGqmvf/xzG3+forKtwhM7tnrVB0HQ1lVq7BnvgCsBdAfL2p9LVWjseDlu5vW+A1/ZNu51vEpZu3Fw+pvQ80T1RlayAVZNj26EOykaePrib5Gb7WAAHQvbcJCuldCv7MfYaR0ZnSJXNFjwPlQVc4Qxm67aeuy09qo54x1boefxqkF60XruNt/gW+f1POI/jQf8Z5CjbWeZNx1Xcg4LrI4ctPZlZrc+j757UU38iZfcxdk3/GuXHusMneBODoxhU2JvVi+dx1HGLFa2Pom8obfT11hN9rrPeUFTKfmH579M/40qPXVpnpp0EWdX4WjA1LawrfFdWW1l07FrH1yZHfgm2J941N9Y6+A6LjY/B2BY+Q49LHQHRxlqUmFVA2NkgcrQRKGSxo8wbqzi0hJucrxH/oVzaXP1VCaecBQf3K9GIJed+zULT4y+Hj1YiUfz4tF9OBLSqbSdLLDaqwM208X7gVHM0vvicph011V69qO++ssidkWlpg4oN9rpTG155W5f/+5iOONYY2WzsGPTjP52J6ShVXvP/Ha+6kfV4vfHafHlxYzzf89zPIier97ztEAVOOKwnAkYPhXk8xJq5rtDJoLPVAd2hstNcdBJHJW44pPRHHGYlhc9UEVhy9GRUjJN12lfbfTwN8H+/O7ov9PX4CxcCsCaNSsJBC2O+KgXHxWdwtOtH6N/z+6Rdd8SuJQqM/r3Ggz4aWlvwdFlHHf5z+O/OTfXuv66pBbMZal7IvG+Agxd4/DENWy203HupFR1X3ZQ51YstDpEvm+T5mHNQ43LPq3KOYhvg/34PDiE7X71s2pbtazGclaoFNdlqyBNwLL5d+BEnLYPS1ePm2T8D1do35zrGsyWnENqrKfnyOP4xaUyH73JHfjUHrHTbdya2IO3EidxUGAuZsmayO3PPPs497wcW7o4WF9CT20NO6Mnqiy6lMxW9F/3GgBGtTKWW/wX8oU5praH7jmOOPTQxcQeix7FRiOgq7/PpLSdDBowHGiWH39pPlc7P2GG+xq6lMys0fPj4JOvYeT5/+Cavz/BbYEL2bBpMwCrrWx8NC5wG/7MfLfNnQC4k9RFzzWtjoE5r2FsXxldeMsfkBftZxXuQ+MNqJ5GHTPj6ZSVEJk8lpOsAqGJoewHQwO2Lo15/vHarMgEWgALjUvNT+mvr2Diq7NZs/gX8JVReHo0+y0aNKr9FMzQtciENqtaryVfwOL4fq1qLVcM/4z3+aa8l82EjmNr3u5MgNG3qjHqDZHWHtoNgz+rZWv1Oomr/CoD1WFo9TaAXrihOOZn7/UHyUlyc8rA6MXbSC8g2+Y3uytleCgzU1hutaq1yTmgplb3PYMOS57lEceLANzumEyGVhzZnt5F0wAYvOAO7jJf51PnbXg0L393vMWOQbMWye5I/9jG+tepfbn7mJ1n1O4Jpwxqy/WHdd35giF2Zne+DB7Ey4Gj+HzeWhYumldzoW/ugXfOgoKV6jxq+2p4YRR8/0ijtm3hxtrLk9fa2fx9yuJ9/2+uEQ6cV3ogWv0DCb9Uaxq3eX5k5Ce6AW2GRKLeMbb+ofryHHafqsneUUUhnPORamK2Q+ZFWPhqx7juWfx52OsA5LvaqSDLN3dD0VrV3b5lf1WfqxuqefSanyBYS3ry1b+roMj8d+p+vT/9C148BNb/UvcyO6hvJOX2Cn/MwY4eugqSleQiM8FFZqKLStTB1CpbZU/5MdRByWfXwSuHw4Bz+G9gLLNSj2aG1Ysp5uEsqMqkhAQsNPqse1MFw9I7qp/Bul/gm3vYVFTJ0AfUB0ck06hFHzj3E+hzOrYVQMfGNpy4wuVR538OH0xS/Yo6jlGTJgC6HQVj1QHVrWnTcRx8OQw4V12N8aRDh9Hw9Z2cb35JWqKHZ4PH8UpQ9QmaFpxIauFcXnL+k3LiWOnaOz5gdgfTdDLBez9POC9mS1wnfmxxHgCDc1MYPuowRrRxMHH7k/zbCGVWVWuEfee/X2Tab4u5KzgJMyO3wc/ZoV07DvU9jtvtJtHwM86Yi56mrpIn29Gx5/YOV8js0D5wt/9cKo2G1Z8HQtM11iX0afD27QoraHH7u2p6THZ2NinJyapBbujAu1PHzvTuMzCyvKPNAJaSS2ZWC1yJaTwWOAUHQYxAOZrDTTd9He3ifZFyjWV2Wx67btJu3+72h17Atb7LsdGwElvs/AHNxJmQRq6+hVTHzvuw7ApXyx4M8j5L0LJ5dOoSxmx9A4C4qs30Kp7OUd4v8GkuApXlrMwvI7+wSKWCu1NxVqqpVYnbVMN8d1xsWdasjBP4v+BIkkr+5P3AVfw3eDjO1FZozjhMy4cerEJzevjRGT2J79a+HUV2PP8LDmJ96hD+zxF7ZXVHc796k6HLHwXgdHM6q9arQGO85uXdS4aRkRD9W/pUPxSvEd1Gb8DiLN+txLXqxcfB4Y1uYG261d9ivE/9HM6Mn82bzgd3WhawLzviyBMYfc/0v7QOl2aRq+Vxh+MtDpqmrpKPKPywxnIl20MZs6FpqgFb443AeBK0KrTQyVM8VbiKVabKoVvfoGtR7U2r2016g68OepFhx1/Kf4P1BwfX/DaV/FIvyxMGckLFB6Tk/xa57wXnE5xZ9lrM8gP0FYw0dhjgUQtTh7nuIbgycnF6C1hvZWI6ovtnh8Mv54gTd/97XaM44jECKqswnkq+L2vDJmcuAC5nzcyNGIYTzQrQ4dU+XK+rY7VE79Y6F09yO3jP8zBV69QFyfb6Fh52NSyAbwWjgZpVJ3/FaZNU+b4nKQ3L1gjGZ6PNfQMzHDRa+7O6COrwQGa30OtTn5G+gEUgaEemow1pn8YbFwymVUpcZDsBOgWWw7NDVNZ+ndTffm9N7ZOWIwHyFuHdFB1KEc7ez67nZP+mw9UJdEF5tFxxx+Es1e0XmUYA2T1qn45mmDD6Fhh3V8PWE5eq2l0seAeKQ33xjn2KdbYK3GqaVmspX1IoQPjpgs386+vlkdsLyn2keJxcNKoDiVQw13Uxo8xQr8aKQpa5zsOFj4q4HJ4JHFd3IEHXIV5lEI0yFvJ1UJWopWmlkaDRdlfLyOITzS/pXX0oww7nbF9ffwjf3FAzUK5eZO03h504oDXnD29f/0J7KWdKDpf4r6eTvpG7153P6YYq5bOrB9XW/gxLP1OtWU54QbX6KFy5036+YdOWbOGg+77mmyVqYvdFvutj7j/DUBdDpTxN7B8qt+PcWC2A8tUd0abTAA+0VKPqd/TNPZDeAQ6+KrYZXTi9du1PaoShGadqSevx0nmD6OpVzehe6v4KLwUnqMlhEJsRZDhVmdprE+oMROGvqFm6Vl2kEXbDr1TVlwq5vdwX8wFshTKmKrxB1hSUc0L/VlTZTrbYKZH00Dwy1fb/MQVcSXDEg5yy9h7yOp7C8JTtTMgp4uA515M0/2VsNDylq+GTK2HmszD4EjjyETAczF9fRF5JFYGgpQ5yep4AiS2ZH2zHBSsO5lPHEWjYbOh0VqQumpJNsOgD+PUFmHqbCl6Nu0f9HtsMhuOfZ2j5t5g//RM2hA6CU9rCyS+rSW1AMmWscZ/J186b8NsGhmZHyriytCJG59WT6bWP0XSdOy48g3vSptJpwweM0ubymuN0uo04DoBO/UcDMM5WGWi2boIVIBjwq1T91V8zxT+YuMT0Op6hpl6tkiNX5ZMSPJTYcbToORKAL+yDKbbVCe2OQSMttG/faL5HYPv6Bj1XwFdFie2hJNA0H2iLNhbz7ux1bFy9mP+5VC8s09z5317/0SfS7e5QkEG3+LvjLbrr6/hCG82WFmNod8UURl0UzR6blTS+wZN/GqND+w5cdPRIWmvbGFfQ+Ebae4onRV3db18yu0nWn+jSGaEvZNnGfPrNuIJe+hreCozlv6HU+aBmqqCRr5I3Jr/JBH6kKKUneR1O4jnnRAAqzFTG+h7F6Yw9Ceo0/AT+rl2J6XTjsP085j+ZuJRsytN68z/neDTLj+aI42vXeDbY6kC6W++BvNTpWXrrq2mrb+O8khfr3X7vH6qpf7g/SvmSaVzhu5p549+rsexb+p1o1cbteoMaM6zepLos5rsv5sStzzTqZxfu8RJ+j9R2mMq0v9L/4uhuT8VaOujqAlYLVCakXktz6rJiVVpbrqcAEKwo4mvXTbwQOJoKr1p+tZ2D5VfHIe0qFpPkz6+xHoAWWZkcdvRpOHToytpalwnL/ew0zppxOJcWPU5QN7EDsX2G2lv1P74uQVvDb8ThTGuD07edkb5/o3tSIvdfPKojIzpn1L2CPcDI6EBcVT7fPXkxAD20tcxPHMUqa+c9q5a1PpkPMi6LfP+TcwR696PqfUyyXsmS7fCIX5WXBKn/88q2LDYu+pGgDc8HVEDZ5YlmDyakZqJrNp3Wvqcu9NhBKN0Mrx4JRetVe4Y+6rlMVOBJlafZmEa019CoLpmRdYYzjZy2X5VJ1dWEOOQB/xlMDqpsGTsuGcryiP/92cj9B3dMZ8m9R9QbNHI7DFI8DrZUK4X0BoJ1BiPCJTL7fNCoPncnw2fX73w5gOKNsOwLWPg+PN5TVTw80JJHHC9EFqmtrGjmLWN49QwVVJyxMlra/8+vlpFtlhHvMknWykjVyuhth7IjK7djazqTLzuE0laHcLLxQySbrFbpnSNfjjdUwPRHBlBSFeDBE3uT6g31BhxwHnao3cYSR+2fLQkus87n0nZaoLbvitshkysQCmf4A9WCRuEqDVcidD0iWvXSwKDRZws2s63Mx6r8cjQsOmgqI/LFFqpi59bAhWr1Up4m9guuJHRfteaM3tLYWmHdUfsfj69cpYJOPh2qHWDT+yS4eDpk9VSN6kwX5C+LDRzdVaT+VTfzaRj1NzrHlTBQX06tdEf06k0tdci8fJjKwtmxSXZ1uzA9DeDsobWPkA1YdswHdDhTderiPL5cvIV4p8mfdis+Co6IpKafMqMlDLtCXUnwlsCXt9NXX8WEzc/AaW+yqdtEKgIa2EECGERKqANVanJc5XZ1tS50paWo0q+uulQUQt4C+r7Vl/ElH/Gfje14f8z3bOt6Fn9YuWodb58OvU+F9iNV0MpXDiOuVYHC50aoiXcA0x+A9arUCW+ZCiYWqZIQrVxdFWyvb+Ec/oHfNmJ6/3SqnN+on+3ebljHdNKCBVRtW0d68WK6ZnvwuNR+FJ+YAoCuqV+SpTuxgPJiVRNvFq3iJ9fVpOzk4mtdXE4Xzjs20qa12v82BFP40erNr1Y3tiT1ill2QbvzmBHsSYJWhVVS9ySx6srcLUigko4r/7trG7gTH731LOmfnIs/NLmo3HZFsoMaylWtGflifw6BpDYkZefiDo3Kvsd/DmszD63r4X9ZIENdzdXiGx7429Piszuw0mqhepM1xfqdDl53PETR7x8x3lANSM8yp3Gl+TGg+vwUG2n4/EG6meqgaVvGYDzJ6awoc7HKysFva5Qa6TV+/2O6ZfPHvUdgOtzEa15ecjxKgmnjT+/KN/oI/pl2J2s7nEGyVs6zgeMijxtR8imb7TS2pfRltL/mpJrqXL5Cvg4O4M3gOAACVeU843ySVu271Vi2JfkEvdHPRH/RRv5wTSQ5LrwfNi7TyBHKNApPThx40g0sHLP/BNabiulwUWk7eTbzTn6w+3GbfxJGsGYD6ApHKlf5ruTrtNMBCAZ8pGllTG91CReWXsS0YH866puxQ8cguuVD28lkHMPhxqkFWfavoyI9ksJsfyUVGxez0U7n6+BA4jQvtubADsYGtHTbYsvmDfh8apu32Cm8GJhAVf6aep/7j1an8GqLO0lo0RlfwOZYfQYOrXH7XFNL738Mx1bdxaGFqoceI66jw0Hjmajdu9PH6g43XkunV9VLTD/uF0bc9jntOtRfctI1sJxrtz+gsrSBf1XVn0my6Lt3afXB0QQtm0G6Oml3x1frF5SQwnyrA6VmCmi6Kr0NX6BM76iOzbYuIX/ANRSggk0VviABy6rzImJmogsPVVyTd6sKRFVuj9z3ZYraN/8Zdw3Xt34Hl6nzYvAYilDb9PpS9Z7ojYsG3dwOo/bR5DvISXLHjIL31ZtpZMT8v99a8c3Ol4Ga5zZr1JTgcHYZRH9W47pHyy7jnAYd0lWGWfXytDUF5byw+VSSChfjCrWlSLHUfmBXFFBsx5Mc58Sf0YMMraTe0jcGnlfjpg8zL8cXsGJLzX5/Hc0KcKHvBrr7F9d4zM6kxTdzg+smFO4H5rPDWajq/5Kqau/V4QyztPbwypGwVJWy1p8pGFVZraeYx7S51fE2AD3S1XO30raRxXYe23Seymo7AEjQaH/mTkLzV5+KUxIbNDJdsW+sPz+tghO+MhU02rIIKoui93c5Uv3hFaieFxz2D1j8IXz7j+gya3+GldNit8NwQvejOdGcwaudVRkLyW3VJLewdgdHp7oZtbzRJWSpbep8eN2vN9xQsrbH1+O+46PT5P5xfC/+77KDaZOmPjRiI8ixB3feQJAyPHTWNjIgFAzrqa1WjbDjQs1/l34GQPyGH6DVAAIJLfHbOpu7nMO1/iujH0qapkrvvr0PsrpTFWrGvT2cmrz6+8jVsaONX3jQ/A8tyxbTddFjnGD8yJvdnlUNyks2wvY1qjdSXChLTDdVOeDWpXDFbGg9KJqxZQXg56fUdA/A7jSeyTk3807Khaww1dUQhzMuMtbW2yJ2TO7+wDKcJOTPpbJwE5oZ28x1RrAnlbb6Wf3Q7iqmtryS8tDVb2fpelIpw+3etXpyiG2EOaBPf7R2wzjNdydrsmP3c7fm5yb/Jay1sgh668jE20GlmcL7wUOwfbu/iTTAcVWfMM6YSzB0lf92f+PLKqoHGV4O3kHbatNCAA674B6uOrnhje0by4jPYInVFkdC8zafrU+S26SjvhnLbpqrhrqhU655MDfMqnFfme1mubsvj2Tez7rMQ+hgrQEgbcUHtClbwEf+y+ig55GwfTFfGtfU+RxmqKzlUH0ebpeTjJLF3FzxGC0qlpNIGb0CSzjViP7ubdPDMqsNvtQueOz69/c1jk5s7nR65Kq9Va4yV1zxNcs4A5qD8uJCfnjvCeavL8LvrcAkgCchlKlQ6zTRujlTcvg52COy/zicLnqPOrFR6zgQmQ43cZqP4qxBXOW9nC7aejaarWssV6EnsNjOpVXBTAAsvzowP9v3Lmcb3zDWmMtqK5t8j2p0bdg+dEf9Ufzwvti1ZAb5JbH71qJnzsLzn4NppRXQV1+JpbvZ6O5EmRn7/lBMItkv9GT266rMbY7VhYvNz6l8bnS9z+0pX08b/2qSOhzERb4beNL5DMZeFjRqkRLPMGseAPOsjow//BhOOKgD399z2k4f22XLF0zMu5+vXDfTKbVhwYtnA8cCcImpjpXMWjLOqispVxcpAlWlDNT/ZFqwP57k6LGkbuj01VdhB3zYXY4gmNhKlZXHZ8G0f0BiS1j4HhU9TmWtrQI55d4AgaCNo46gkdth8OKZPVVvLX85LP8qct/BF6geoBvjurCiKrFGsODN5SYzrZ5UubMi2RENrV7NTnKTVy3TyBesO2gUF2na3biLpvscq4HT9VJzY/sfvV6zzDl8Ufiy0Z1q3P7hZcPYWBR9f3CaBkF3Kh6XiRO1DSlWEQDl29aTZ6fSItlNp2Uv0l1fV3+mUSiAxejQtOKDLuAm+5XoNrUbHl2282GM1uepr2/dWP9rrubLa0dxQyN6BO1rwvt7uJ9iYShIW1RR7f3joAug39kqM7CqWFVtjLsHTnypQc9REToP0zRIDP06nwiciCdZXWS81vyQbvo6gmiNPu/cV0nQaH+W2p7SIaEx7batAkZJ1Q7MTFc0Syjoh69uh42/qz43njQV7Alf/bNtFdAIVEabY/urVIlap3HRdb52FLx5Uux2+CvB4UE33bjj4lU3++sWxqb49jxeNdhO71z7J2piC3WVaMjFdb/eEdep/kitD2rQj6e6+NAb0DlD2zGwXSotQw0QnUb0AMDaob92QZkPsBlrzCUu1My4n74SNs2FgefDaW9GJjSFmYaOHxNHyVo6ahtZmj1BNaVGU7XKnQ+Dox6lONStvzAcNHImqPIzwI2PdtoWRv96IcnFy+iob6Z3wRfqd1i4Cn5XPUkiU/AMp8oyM53quRxx0ays8JjK+HS4uxj7zPcYc/wFnHr1o3xkXc3f/Bejp7XjWv8VnOz+DwPPb1wDuX2C7mSksYixxlzVp6ua6/2XcX1o1H2abxOpFWsoDzWW9nq9lGgN6y/UEIefegkTLryHNe4z6Vw8I+a+rvlT+dB1Fx7NS6BaEGj1+rpL1eI2zeQ0c3ps4Hg3yrJUVlrAr/bRfzj+WoZFolaJacQecA/rmN6k42A7ZsTTXV+HO2knDV6bkWnobLAzSGvXdP3EPHYlg/OjzX3nelTT4Fv9F/J1+pl0s1ZgFK/B8KsTtrad+uDyJJGoqYPqNe4e+Kg7C9RMacnx3nspJw5N13Hjp0dwOZcVP05O8Ty6BZbST69WKu30cLY5jY7+ZcThjelfsqMPEs/C3fMoxl37H35xDsMKNYD3eGr+bfo1B7/NnsmoP+7iqmc/ZOGaLfhwokX6RDTuBN7jdvFC+3+TlN5y5wuLCDP0PtubFVxg/o/xxhxeTbikxnLGxtl867qRQwvUFd5g0E/A1skObOZ2hyopnWV1Z2OCuvDzsfsESjPqb5zuiI9Oc9xUWBpz36aiCh7xn8oSqy1ZWhG26WJq1iT+zIg25j3JexfP6SqAsi2oyomnBgcBEBesOYq7uvZ5Uxm//R2SXAZnmSpjoiElvXuSx2XyjOPfHO59iG531Awk10d3uLEqCmipFRCfkLTzBwCHnaKmzoV7VIWHi9RlXeJAPgkOiwQQS/DgdtX8GSakZsGI6/Bn91Nl+J3GwbZl8IsqQW335vBIhni5N0DAivY0qs2IDulqAAzARxfDtj8BSJyiSnSLE7uwYENxjSbVJgGG6YspyBpKauizrKGx6aQ4R2TcN4DXb9WZSRTOUHHvz+VpoI5lG0I34PhnVf/OOoQDcPGumj/TVilxbCvz4Quo0sX0QD5G1XYMy0/AlcyK+P5Mt/tj2zbrMkZyo34T8S6Ton6X8GFwRK3NyiPC2VL9z1EBovhMhhZ+AoQuVB8SHXrD1Fs52wxdiHc1vEy/a05indP59gfhAOxKuyWPdXiZN4OqnH57RbWM1eFXw/GhknPDAX1Ph4MmQv6SHVdXq0qf+tuLd5oEQiXKR1/yAHEJKQD8qeeSSTHbtOYtKd6T9vN3lwOcJ42qrieorzUNLvleTdUKu+BLVd8NUBGasBSognM+VJk/hlMFItbPhj8+hi2hqRM9jocB58Fvr6ggUumWms8dbmZt25DVTQWsTKdaZ/+zay6/5FP1RnrVbzXvA0hqBXPfgjmv1/16SzbDrP80OPWwuvBI1bDwVITqmUbBHT7pR3bJYELvlnhtkxmWKicKoqvJD31Oh23LsTWDpwPHEUzJBcCha7zkmUSVI4UrzCmsj++tmor3O0tNT9v0Oyz/ktJQimVlKNKNrwx+VE3Nq3BiaBaabeF0uWmt5dM3/zMVBCxV5SOM+puakAehCLityv7eP1dlHKWH9oNw8OjffWv0t/JqbpK0CpwOk6H6HxwRjM0C2V8szj4m8vW2zKEx900y/0eBrQ5+e2/7guFbJ1Ng5nCx7zre8w2j3Gj45LTGSPPFXlHSDCc52naWW60pCTVJLC3ZTvuXe7FxQx39NarUxAfN3zSZRs5QA/bt6QP4w2oXCSDsisf9J7HCaonh2vWsrV0R5zIJ3LaFniOO2fnCzaj1PSvp1KlmudXuYmoqIr7Mas1bgbFYoRLhp5xPc3D5NxxT9j4tNn2Dw1/Cr91vp9fwCTirjZUPVpXi0+rO8HCaOn31lXhDgSXDHYcTHw7bi+n0UDXqDr6uNnLd61GNyZ3J2TwaOI1KX90nkqNKPiOz/E/cTgevt72fXxLHUm67MB01TySf8FzNknIVTPrBdR0Zi17Bp6nlzvPdzNtZDeyXEWLoGq9fMLhBpSYiykjOocSOI83p5xrzI1ppBQwv/6rGclYo09kZVJ/pla5MTrHujwwGmJl0BM70trTbPBWAH40hBFPqb+zqdLqZF5r+Flz6Rcx92dp2jjN+5uXgkayycpjd8hz6ls8ksyB6XDJCX8Qyuy0ldhwbU4dgWxaTTNVX6yx7JyVcVhBbM3GZBteaqvF3Y0t6m5qmaTgIMkz/A3d9J761MJxxkeBvfGLqTpZW3PGq7HaZ1YY/rVa47Ng+mVX+ILm3fEZRubpdqyjgg+AovKGTuBOMGTUaz1feVsiww06FX57Dsfk3VU42P9S3bnA0ONlSU1nDZd5gTE+jGsry4bFOavpS2NY/1EXXUJ/QlnFqe/xB9V6aHq/20QAm3ateocTTNnJcaTcwamTqGg/9b2nkeLC+TKPwugPW3pW5tlsd9Rgc/XjDl980T7VqCBtyKZP8N0W+DTfCjneaNUoTww2xS6v8VPqDDNBVkBA7yNd3n4U58XMme0fQ/tYveOGbReS2Cg3T6H4M1/svr788bfDFaljN4z1UZcAvz2PafjQsXARjL7zbDRyXe4AJB8SqcOIs30xuqN+QNxTkA+CpgfBcKGvLdMH8t9UE7vdqlgfWxhtQf8sep0FVwOaHYG865aTSossgSmwP2Uke/Bhs0TN3sqb9x971aSV2L18Z6e8erSLzm+bWrAU2XZFpUJRXax654D1Vynb4fapB8svj4P3zo/e3HwXHPhktB5v/ds3nDmcjaZrqg5SYo9KCSzfXzEQCyF+qmtYt/rj213LEQ9CtWhPt2iz/H3xxY7SOtRHOGhLb1yj8hu+qpRE2wPQbR3N0n5Y8c9YAim/YyKhDQv00MFR53sL3YNq9/JZxLFOCwwkOURNBTEMnELSx/VVY6AxY8yIseBfiUpi1vgzKtsD8t9X4cSAQbnp08XS4cjZ/a/kGrznPwCSIZlvoDheJRpCAOx2unht9AWNuj05ZaHuwCkx1GquCRMOuUEFBUL+fcCpsKLAVoenc43gdl1XJeebXjAw2fCrdvqQkUQXQVlk5VKXHZnNcbH7Oe65Q+aVuoFsB8gLxrLZbYPjLuEu7fPdvj+0hvdf4mNu00N/aS8GjKIhTJ0X+KhWkuf6Z92OW3fLbJ1Te15ag38dSqw0fZuz+bbQtC8NWgeHg1Nv4b3Acm+xdK/GabXUhWSunhVaA6Yjb+QN2M9O5ZwNVe6N/+U8G4EnHBSwacDflZiqfBVUAtUfl71iGG9tfyQKjF950FbxyuqNBo24F3+DX6s40inea3ON4nfeDowBwOD24bB9Oy4fp8jBg5JGMv+rpyPIb2p1Avp2EJzWb54LHUl5P8sGhFV+SXKECp22C6yisDDLSeLPWZdd6ejGa6GfIVq/JJc4HAdhip+Kwdj3wKRrOJEiSVokrJTq18OiKj2ssZ4WyKsMZPD7bYI3eFiuUwevoPIbWegHJZWpC1p0l95BetKDe545zGqzMUc2Zg97YC0zbtAy66ht4zPECJXioim9Nn/IZtCr8NbLMdY7/4z7r3/xmdaV1YC1VFWX0DQVKVuu190eMvqAA6EYkUFRo7/4G/7tDmebhEscXO19wB7ozOpnQ6WpYs7+4UO/ADK2YzvpG+vhfibn/p+++YK7rEorWqt9rq63f8obzYXwV6ndXYNfMKIxzGmiahrZympqelr9U3XHqGzA+Gtg73viJRN3L//2+gY/mbsSsq8H7RzWz4PCVx1ygvOMIFYjslJVAt5xEOmRGM+krccf0I2poXCccgFq9TT1PuTdAQi1ZMWF3H9ODifvoNKwGGXwR9D654ctXFas2De1D08UGTqStFr3IHQ42xrtqBo1cDgOnoVNSFaDSFyRZK8fqOA7aDoW8RWR/dRnPOVQA6+BVT3KBpsorw2VpdQYgAZJaQrdQuVzvU9SgGsBJgF6vdlLvE2d9UPfjReT3lUMhV2+5g0va5/P4aX0B1fsLUK1UtoSmWh7zZKh1x0aoKmrQc4TPvzxOgzI8XGXeCQ43KemZPJF6O87kbD6xhvNo3LW78ZXt3SRotD/TDBzb/1QlZks+jY24A0w+Ff4MXd1zhD7sfeXw4UUqs6XTOBXsOfIRVYYW7imU2k4tG+6PVL1PkidDZcyE64595fDL8yo41ecUGH4tlObV3FbDqZpq/6+OUbhVRbB2RjRQVZtdbIQNcNXYzqy4/8jI9/GhK2zVr+qEY0YPndibtmnRg6OsJDc3H6FOpFaHJ4yEGktfMK8LjzpewEhWt5uGxhmVk+m46r8EbR23txC+vY/gzOc4dVYnNg28CQxnJFIeCNfEtewPiS0pL85nc4tx3Oc/m0B8Ngy+mB4n3qL6APz5tQoA3bRyh5+LA7J7wQnPq5/R9Icgr9p44IMugNaD1SjQagKhK/BOtzqRr/C0atTPdF8xYKM6weyg59Fq+68x9/2p5bI9fFBqutAsH1l/vMrXrr/xmOMFtgd3f8Ah6Z7NdO49OPZGp/odXGFOIXWjyvjyedUJ1SXmp3gD0atRs3/7mbhAMXagChsNrXIbu9uMFQVc41eB0EMK3+dBx8u01Ap3aV2D9OVcYE7lqcAJBLL77s7NFA10vKF6LGS17cqDJ/bhs1bXMi2oynxs3SRoxEGgkrfMEwi0Vn3N3GmtONarAqoLHP24Je1fta8cMAyDBVZ7FoWa9huprbk5cDGFJGLG1czWy/RvJFMrITE1i/+6HqZiay1TPsPrtv3ooc+Fsds/YOSmVxiSVFDrsreWPsDxRrT00/ZXUm6qjIiprls4Ob9x09PErnGZOmusbNJbdwGgwnbV2ssm6K2k0E7kD9QJubtwKd9qlzIv41iKbQ+aI05lxYUuUuVaa3BpO+95ctLl9zM/bgi2P7ZZ7vOu6BXofvoqhm95A8twqH6B1XRjDZU4SQ/mU15WFLn9A+2Wep+3wkii1KmuSm+wM7jUd91Ot7U5mNfOw7zs+0Y/rqzNaDbbafxxRM3JhXWJT1J/f2vHvcAPh35Ab3sFc1esZ8XH92MHfKQufJlUrYzKUvU37SlWxzeVehxPBY4nXaunXEkPTU8L/57/70J1TBoqWZpgzOJEVzSIXKMR9pzXYNGH0ePZcXfD4SrIjK9cXZDVdLh2Ea4UlQHcJtXD1GtHRTJ/wnxBC0ctI97rE44tlVSq5y+tCtTIiq/u/OHtyUzcxckc+yNHnMoOW/29av3w7BAedET72QSC0UySSKChmqQ4k9IqP1X+IClaOVrBclVZUbIR9/ofONKYTX/tT04zp2O71OdYaryT9y8dRutUT431xehzGhx6u8p8WfE1s1pPJIlygnEZ0GG0qlg4//Po8lfWUYVxgPOFehqlpyQzrnu2ui30e2XwJTD6NvV1Zld1Hpqaq4KJ9WT7zV23ndxbPo8kCrgdBgO1Zdzo+jiyzJ3XXol74sd019bSwbd0t7+uvZUEjfZn4SBK0AdlWyFhh5Gppjva0yi9o2oO5klXH4LxWSqF748pUBhKyR19C5z+VrSHUWtVxx8zPe1vK+Gu7dFJPyu/g6k3q3Vu+1Nl1ThqeTM1nOpDuK7paHkLVdCotslqO6pt/Q1gVvtA97hqTqIIv4GcPrhtrSOHrzy0E3Psrtgdx4A7BYB7HK/RT1+JMe1utWm6js9W69xGEuHpkAEbMikiftt8MByRFONwxtEDXyyhJH8dz5TfwOlMZa7dmS0TZ0HuCMjpo/69fTpM/EJNB6lu8wJ4rLMq3TMcqqSw+gSuT69VjbJ38EqKCgo4HS5Kr11BzysbfiC4L6nw28y1VLbRjgdkyZdOpfRi1dchEJdJhe3GWRRNUX/Dvn2PbGNB2yP51epGG30b7lL1u/J51YHwGGMeVWXFrNhaxks/rmKF0Zk1VjZrU4bwu9WZM7c2Ip07ZM22cr5YuLnO+zcWllCVMyh2G2u54tsQRdevZ6nZje0k4knYfT2iRMNtSu4HwNg8dUDd1vsnjzufA8DWHZTGtaZES+a80hdJ96srtXFOkwAGPwd74PJuI1uvv9dEhlbKs84nAXDFp/B5cAgnBh9Ay+lVY9nk4mV4bZPErHZ01jZRsW1dnes17QB6aHqZbTo5zP8d51sf1bqsUa1n1lKrDUN9v/BM1a2R2/bfAcV7F7fbTe69y8nKacliqx3X+K+oNWi0PnUI1zrv5rrgVQAEA34CmOQn9aSv9yW2tz0c23CiBXwELRvD8mM4G5atGNRdNYJGh1sqeLrRTud/wUFgxmHrzhqTceI1LxOMWeiWj8rS7VTZDmYPezaa3VuHWdmn8X1LNab5T6sVJxr1TwZsLskpaWRm12xMvjMePUALrRBnXMMzqFyuOM7Peo/BBw2hb5/+fOS6ixmv3U6neY+w8vfv0EN9Nb0l6qJEZomaIlXy8ytstDOYn3V83SvXDDU9LTzwJehTQaQNc7BH3ghAl5To4jUyjT69Br57AE58UX3f5zRo0Ued1Kd3UhdSR98GKW2iTxlaRY2gUSjT6Kkz+jNxeG6DfjbhMratpVVYlk2ZN1B/g2URq/pF5lBLhuo9s9y1VBRUl+h2UFoVoLQqwHYtGa1oHXx3P+QtQItXwd9rzf8DwG9GzzsG5TYg69qVoHoXHfYPOPJRfml/BZ31jdiuRDh3irqImxTqlTf8Gsjo3OCXfaC497ie2KGhPrrpilzk9wctZq8pJPeHQ2B0KBFhyhWqp1lqezWoKODlvd/W8/asmscWv65W7zVFFX4MgkwtOob/c93DOVVvRReqKISlX3Cs8TPjfdNqrGN/JUGj/ZluYGu6OuCpKKgZTHAlRnsZbZqn0vbcySpgZJgqu6eqBJZ9Dmf/H/Q8Af6RoXrigCpdO/xBCL15YlnqqszcN6PN6harun00TT3H4g9jG2CHtRkCPY6NTvXaUfjNs9PY2u+H6NWghOx6figNU3t5Wv2PmTSiPZ21DWgrv8UONazrqYWCMaGJc6ah4bN1lrQ6mUcDpxOuPvMHLUboC0le+xW0HEAwlGEUzjR68YdV/LBCvZG1K53DR847SV/zuTqg+eFRdVBjBeCHx2pG0MNd/ee9BeP/oYKH1Tv9Z3ZVVzx24Ayl12q6TmJKJk73rgXj9naOii300lbjsw2Sc2JTu7OyW9C2ldr3NuceT1LJMgZu/YCfM08FoFzbMz8TT6CIR/ynsdjsGTnRKY9vy8sBlR3nryzl2SnTuXBaf/rmvU+uvoWtwSR+tzpjWDVHWe/M/U88Du/V3UCy3ep3ubnqcS7I+TBy2weh0qPGSklKwtY0Hnb8h+TgrmUrib9mxA1qvLaRqN47B21XVzinBgfxZ8Ig5rY5h+lpp3BCYCpJhjrodpoGX7hu42DjD4ZXTOOCshfqfY7A2R8yd8TzALiDZfzsupITrK8jQwSqq8JBAUkkxsdTaiTjLa6lb17Id8bBBEMDHmxdnSRYcbUftAcc6n35Gt/lvNTh3zwVPDGSUblw9MvknvN0rY8TTcMdl8jV/ivprG3kZ61mA+tiPYWKpPZcYr1LwFeFHVRBo+FbJ/OO8x94rFJWZoxhYdIopi7cRJZWVK2pef3ebnU7v7c4NWas8piACuK00go40piN7ohjS2JPNrrVCZttWSwMZcsBaAEfRX6DT/UxxCWmEWfV308xu2QBbSpVE9b/BCcwRG9YQ9Z9RVL5GgCctTShr4um67x2+eEke5y4QscYV5pT8NoOqqoq0EOfX/n56j3gV70fAAOWPc5Djpfoe3ndfS7tXifiz+ylLiR2CpV8O+Nh62K0UBBhtLmIU9qr9zSztkwgf4XqiTTienVRNXeEOqnveCh8do3K1A4xdI1RXdTx8MC2appW2BcLN+MydY7p27LebKHqwonmJZV+flmtjtUT9vfpaLtTcjSYxy/PAuCqFjQa0DaVb64fhaZp/N9lw/h55EKVXRaS4nGQX+rlqCd/5G3/IdB2GPz+uhoKFJqQvMpWJbaaFX0faZSB58GQi5mw6DoO03+DtA7R+9I6wF1FMSWVIurcYbkketT5pB2XgiNULeELWPy2ZnvswqEL+eT0hhuXg8PN3z5YwK0fLowsUlDmZfhD30ayzvJKqrDqupS0fQ18ciUZFBNXy6TW/ZUEjfZz2496UQWLWvSFrB2m77Q7GDbPU19vWw7f3AVLP4Uuh6nb9FBadvk2+O/xqi8SqGht2LDLVXf6YADWzYQPJqqIbtE6FbxoN1w1rwMVEErIUSm+O2o1QI1GbD245n2gpqeBKtOqy8Dz1fS03dBYMpxGXD1oNKJTer2PcTsMckIlOp8XtOJm/0VstVNqrNdrmSQVLqSPtpLvk46H7F4ELPBjUpE9EAZNisk0CqfQlnrV/4amk0Q5nk8uVEG9gj/hV3Uyxrf/qDl9Lpy9ZbgAOzRJrdoVGH9lrSWDQ/yz+b/giHpf8/6gwtMKhxbEqQXJbFl3P4C0wNZI7wri0vjj+Kno532yR7YxtXgJ/+e6B7cBduiqaaB8O7+6DsZvG/z9qZdpv1b1Njo0OJOXAkfSb/m/ucX5Dobd+KDRaH6nt7a67gX8lViGmyfOG8ktfnXl/BLz87qX34ni0LjUhjZPFbuf77Z8hlyisou00InaTf5LmJcwkjbeP+ky7wHcmp+4JBWQCfeDWG9l8hmjCJr1B1Dbdu5L/3FnAOByuWmpFfKg42Vc1Owj5PIW0lIrxNA1Khyp+IrVpL7yNbOxqmIzml7UTyOYFipzcqspeJYrudZtsJyJvBw4kjGnXkm/zm0JWhY+Q2Wm9B59MunZbWp9nGgauq7xqOMFjjR+5WF9Uo37u616lZsrH+ca80PKSwqxgl6CmonbrmKovoTkkuWqEb+zN+aWBbwZGEvr7oNqeaaaUrRyypdOJ+6BaIBRt4M877mUqWlqWIfujGNNiyP5lJEUlvvYXhngPN8tkX5fv67ZztyPnuDTNjfiTkwl3q596MD8xyYw5+lz6bHtS3oVqcbJpxnTaa/XHQzdF+nOOCptJ87MXcuKcFUbhLDQbk8VLnw4eMB/BrcubY9t2zymT+S1wGENW2Gvk9X0tA6HqInAEM1E/1hNbWuV/wMP+e4jnWLKKmr5/RlO+PlpdcwUPm5a9CHMm6zaPjw7JLLoygeO4qje6lj1ghHtOXNwtMfVN0u2RhovN1R4+EqFP8iZ/1Gl8wmNbE5+QPOkqYbTo6PZpOER7WGdstSxx8B2abTM/wltQ3RiYL82Kfy+bju6Bn/vsFw1PwcYeQMMugiciXjw8u/ACbQcfsZf2tS2RbMYri9G7xzbz7LWadIiQjedDKx6juLMQei6hsPQuOC12Tz/vSpjtSybaUu2kB+Mo7zbycwpjodlUwmWq6CSRrQscVleKRuLKin3RUucbXR8Zi2Zk444qCjgFPMHxvZq17Qvci8iQaP9nK/NSPUhOfoWaD8y9s6Dr46m3YYnLCVkw7FPqa8NhwoG+StCab2hDBa7Wu1v0TpY/iXM/S+8pppL4k5RV2buSVGPHXxRaH0uKMtTpVQ7Wvmtah59wnO1v5DwqMmK2ntV7G619TQ6bVBb1jw0oc7HuB06muHAn9SWeVXZjNPnYBLkucAx6oMLcBgaz+unMsvuziWpc1ijtYALplLY/Sx8mHi2zIFlUyMN2AJBO1KfW+JVt8V74miRENou0wkFK1Xzcqi9fC+cVWQ64afHwV8eewVm23JV+rcDzZ3EOP33nf+w9nGHT/w7W6/fwpaJv+L21J1Wn1GussVu9U9iZevj6dFvGG1y90zKsGao/XFR0kiWJQ4DQM//g7utp/Hh4Dnnv7nK/BiAMtvNGH0uuq+YMj2ZgN34t/mzzGm00fPrvN/2V2EZbhLdDmZZ3TjRezfLjtr18sWnnJMI2Hq9P3/RtJxOJ3ropMYOXTVd6L6Qgws/Jrd0Lr1R+39SWuykkDZ6Pq386wg6askgrYOrWtai213zcQOOuYTZY9X+ND3rHFbEq4sF616ZyM9vxV51vdb3Ap4KNW1wfpuzeS1wGIH4FtRmftvzWWK3JTVYSNf8r/iH4zX8xv6ZQbmvGKCvoIu2gav8r7JmWzmbi6sFEQNecMThsw0qSrazNbk/V8c/gha6EGI4XfQp/4Vj1z1EyoZpdM5w4XE3rM/cuII3OSP/iZjbTNvP8KHD6HHkxQBUdDyCrv6lnLrqdl579TkKFnzJfY5XmGN15svgQVi2xrn+9zlO/wlXdleO8d5X62Ss9NKluLfOQ7ODoKn38mP0mbvw09q72ZpJnOYjzrlrpxbVJ8klUsHW1P48nPx3ssZeyW/6eRTc045Lfa+zwOpQz1qqmfEE7hVfqCzrzaEG6ZoGZ7wDHQ6FYVdC60EYXQ7nI+eddFjyfM119D0d5r2pWiyE5S+Fjy9TX1fUnR0b6a0SUtfks7qE96Xq2XC1tUYQdQj61blF26FqAnOP4zjed1/dDc/X/KgCgSFtUj1sLfGSFOfgNP3baOPzyiLVo7Xjofxpt+LxwCl0yv1rgYMqM5ECktCrT7gWO+V0GBSQHJn06DB0luaVUlypMspKvQEmvf4bj/+8nbKVv3DScz9jfXYd3/34AyfoP7LafTZVler3qmkaKZRSXBHNRkumDGdADWKgyxHRJw4HkN3JGJkdm/6F7iUkaLSfy5g8XvUD+uoONTa0Ol8ZrP5Bfe0PHaitnwXzVakC4+6BHscR6fTQ8VD1f/WDos0LQqmaKdHb4jOiH6Rf3QGb56uvw2Pe/xcdeRlRUajK2hbWMzHgzsI9VtebkagOSqv3NNoZTdNISkjAUbKOlt4VjDd+5xtjOJddeGkkhdnUdVIC22jlX4vb5eCYLc/D5zfgraqgODxJZf0vkUyjgGXh9asDjy1WMsONt9AOvRWPHjqIMJyqXj8uFSZ9rdKnd5TWQTXU6zpBLT/w/JgafG5eA2fWPOFPdgRI1ppmXPvexO0wyEpyk92u/pHmZqhPwxh9LnZi7SelTcUKqt/3uvThrHV3BSDoq8SvOblbj05H23zVWhK0KjroeZjeYmYmjOf6+Acb9Vx+nyoXqrDr6R8WqMAy3WiaxreuGxmYsI3OB42ve/mdGOL9BVOz9rrx0weqVc5uPOQ/HYAcowTNGcdA/U/+m3UjnloCeycYPxJwNTxLrPrvubZAocvpYtDIwwHYkjqAjYYqP/vKOoiyzSsiy3kDQQ63fsRtqfepnMBGplt92dKhlgmdQFJ6Dvebr5AV3IQZl8AiK5fXW93T4O0WTcOlBTiHL5jyxBXc92K1vhFBH7bupEzzUFlWhBWoIt4IRPpYmA43yR6TdpVLcBSvIZDQ8GENtuGkzHaz3U6IZPNO0w/GSmqF6VCBp9ZZGWT6N3GUMYuDtrxP5y/P5ihjFpebU1hqt+U6h+pn0qpqBUnx8ZTacZRX1czs/Cw4lFlWt9D0NHVys5KWu/Sz2puFA8oex64HNsKTHLvqG3AVLqFT1QI6JKnjoQyKmcgn/MtZS3CnFtrWPzBK1qH9+rzKsD5Y9cai65EqILn4Y1WuNvNp2ur59E3coS/b+V+o8egQGzSq3mLh8rqnyqZ6Ytst1NU7py7hlggVviDZSS4+vPzgRj3+gGc4wBGvKhzSOkCncRyp/4rHqNn0OqLa7zneZVDuC9DZv5yEDT/ASS/DIbfAby+r854ln9Avw6Zny5rDHBorL64TnweHwI6ZRqJe4UBs+Fxtx3eeklDw6P3gIbza7UVAo1RLIGfj1Ejfxu0b1TFFXMlK5rkvoaha0MhBtbLDU6qVwoYanzP2Luhz+u57QXs5OULfz2mBKihcCT8/Fe35E1ayGd46VWULTb1FTUhbNxMWhQI3GZ0hIQvuLoI7tqoeSH1Oh5b9ouswXSoLKXekCgpldoP2o2KzkcIT2jI6qW72tU03MxwqiLXg3bpfTAN7FewObdPUQYGjvrGZtdCT1UGrHvRSbruYp/WAX56D4o2R9R1m/cTg4FxcDgdW0A8L3uWTN57gV7s7+V1OD01PUz8/f9DGG6qvLaus5Hj9R/Xhd85HqhH5gPPg6MfVB93ij6HL4TU3ygqo3+3gi1XQaM5rULwher8rARw1r84OOPUOZo2qu1/AgSaoqzKW8cbvdM37bM8+d+it+pDS/zF846vqNm8Ffs3JT85oCWGL9BSmZU/EsjXifNvweDx09S6sdZ11KcxTjQGvdd1d5zLfZl/Aj+2uiHx/nv+vjYftGVzCvwMn/qV1iN2n69HXUD7oKn6xe5I+7Bz0UIp8R2tNzHKTfDeQZ6fyQmACsztcUcua6naJ7zrmWR1xOOroYxcyrmAyQ1erqWbXmB/RJfhn5L4Xv1+FgyCGQwU4OwZX8ZrzUTKpPSO1a9EPuDQ/LTr0xnAl0E7bQoax/wfG93bzrI7oms015kf0C1ab7Bn0YxlOvjDHMf3jl/BOe4Dbyx9mfYsj1MQ1dwIJmW3IsPLJLl1EXG7DStMAMF0sow3X+y/D61MnCc/qZ2CldSIpoyW/251pneIiEFDHTaMM9T463T2OTK2EFKIBBttwkuA2+M11GeUF62s81WnGdLrr69nqbEuxR5UsTbcH8EZg/zpBzMpuCXcX7/Rvuj5Ozc+WUFl/0rZ5XFL6LOkliym6YgknVgsWVdkOOlS9Wf/KNEMd/xSuUv///FT0vtJNULIBVk2P3JRtlMU+3lui+kTenqd6GkU2slqgO7NLnU9//sG5fHfj6Mj3nkY2sR7VWfUhrfAFqfAFiXPsuWPg/Ya/XLXdyO4FS7/gGceTtDCK61zcHhq9COdxmpR5Azyhh4aJbFkUmdRIwUq4YjZHX/k4n189spY1Nc4PWWfTQ1ureuWIBguXfLod6v9yX2xvqZIq9d7ux2SzX2UVLy3SSS9bDkCRHU95pUqasH0VFNiJeCtKOE5XQxGmXBYtP+XXapUw8Rmqx5XrrwcM9yUSNDoA6B9MVF/smIXiSVOTtJZ/qb4//zM15tERmj7y8eWqyfLCD9SoQoATX4ht1GY4Varuox3hgi/hil9VEKPn8XBBKFjU57To8rNegIXv17KRoRKq8HM3s4HtUvn+ptGRvh0NlduhC2VGEpVBg3jNy428Dks/UxMXUI0WfaFyIW9SbmQ6WrkvSColpK38GPTo9LRA0Io0ZasoK+Em//MqIyu5NVz4jfqdJreB3ifB7P9Er6RV5y2Fl8aoRoDh/kbVJ97VIT4+nsFjjm/U69+fmfHRPil6LUG2plSc0JEZwZ4kUEGST0018+Igz2zF0/67YpbtdvglWGjcyPU4sjpzR1Xdo9BrM3vFJqYH+7JI61rnMnZlMR4jmnFYvS58VzgsL5ls3/mCYo8Y0DaVfxzfi6H3/EzXnv1I73c0AAXB2FKy3mNO5xPtUEbqi8ixtjbqOb6yBnKS7+6dLhenB4mr2ooVOnk3q+1rPda/hUeLvpfFp6hMh2x37WPX2/YaRrkWT3J6C5xxiSRqlYzdXs+FCtHkXg+M5xE72nR/VODnyNc/Zp/Nj60u4q3ESfS1/qCltYWg7sSf0IIe3lfRMjrRputBmHaADYFkktv1bfDzBh2JaMEArzofxR/qb3Gj/3k8VXkkJCQy4J7fcLnc9DzsfL41ohkermNUj8bzzK+jKzOcGIZBOW4qimuWKy212jJFG8O3aafyZ476W0rJaEmctKep4Rh9JtlaEaDKoE3bj+6IIyWzJa3iollcbs3PcXrNsvoYKW1xbpmP5q+A/mericFhfWvrQVMtiz7oVxNpf3lGTZitngXbJtR7s3ogqRa6rkWGqgB4Ghn0OX94e246vCsVvgCVvmDMukQD5Y5U/xKywJWAZbp49Pg6An1th0VaSQAkOA1KKv2qxQTAmhmqxQOofSmzS60XXHeFq9MoRhvzoXT/6nPW1HbMNNpR+KI7QGG5jwSXyQKrAwW6Oh++0nkfBXHqnNYuLyRdK6VFxVIecfyHoR3SaJVU7U36u2pZ+8EApOZC22pBpQOABI32c3pVtROxHSeThQM0BaphGEVrYf5kKAw1wNUN1ZPo/yapLKDaVJ/I9uvz8OXtsPI7+O0VNTHs7mJIiTYD5LS34MxagkYt+qrMGXPvCBoBtEtveI+OsB5JlSQES6i0VRAsxxk6ofGWAGqkq9cymG4PYFOn0ykMpUHaaLTStmEEq6D1QQRDQaOCch/5ZWodm8ODWTbOgX91V1fNFrwLPz6mGvMFffD9wzU3KtzTaO0MNboTosEj0WA9ew9kxiGqdMJw7dn9tE+3Ljyf+zgYTrSgOiFen3Ywz2bcTmroivchXhUc0uMS+c7qx/aSctIyc3BUmxbSEK8u8vFk4AQ+9F5U5zJHbH6GAfnRseY+7a/tT0NZwJnmd39pHaLpdOzcjTLbjbsqNjB07bgudO/ShR76WtoX/9qodf7oupY3HA/tdDnNlYARKKe8VGUPXRYfDYJ6KvP4w+hG286qT15KxyFMDhxKl+61D0zIaN2F+Ls2gaZhtTqID4MjsGub5in2mLjjHue2S87nPwHVE1G3o1eKfQEbzeHi9MAUBunLGWUsJKg76ZT3P9a4z8SJF6fLzR1J93Gh70bis3Ib/LwL21/AD1Zv9TxeNVzgaPsH3HZVzHKJCQkM/dsnfDByKh2r/kvrFtmsHP00c7NVCeS1vsvZnK5OHMq1eLxlNYPfB+nLOM+ewsC1r9CqTPXWOfHKhzn+77veB25/1U7fyhy7C7/FDcf2e3HYfozQifnF5S9QYsexyVZNrcPlJXWx+52Fc2MoCJnRGVodFL0zPE0p7MJpseUn1S+sBWL3CXJ6q9HdHUbv9PVkJLhonaqOFxqbaQTqmHHKvE0ELJs4CRo13vmfqZ5G0+6Bhe9jOlz0yanj+G3QhWi/vRL5tv2qtzi47GvVkkAzYttj1DXpeRed1d2khVa424JQBwpnKFgUzjTaka9a0Ki40s9DJ/XmEfsc/meodivjjTlUhHoaWaGp3/FVebg0P4auqcnd/c9RK+hWraetpsH8t1U24wFEgkb7uaLxT6gvznin5p1mHGR0jQQ0+CA0OjTcJNlwgK8i+nVtsntGJ579+C+Y+bRqJvftfTDrxZrLdz8aWg+seXtyK9WYu92+XbOdEKqVzieNmS3OIbfX0Jj7HYZOsR1Pb/6kn7WE5wLH8qvVjSAGXpwEzHjoeGgk0+jFH1Zx0nPqoGdrZegqmLdMBfG+ukOVAa79GaaHTr4Wf1xzoyIBIi06Jc2sp1+NqFNWbi8A9D2cEZcW7+S/k4aA6UC3VRDIXfQn/fzz6GCtYVqwP89epXpBGPEZXOu/gpnuq0h12jjtxgWN7vA+wa3Zv+CuZRR6mB6oQgv9DJ4InMj/EmrvIdNQm+yMnS8kmtXnXe8nedwNNW5f1/FMZltdMOIaN3Y2i+0MNxbvdDnNnYgjUE7lUjV1qp29gfV/LmDDn/Nxlm+mpN1hkR5JrVtkc+Z9H2M2oBedx6HTR1uFVr3UROxxpw5qQ6/WKTwcOJ0lVptIKS7AYRufYnjeZPoEoiVrfsNDfIa6EOUKqpP5grj23GG+SbKr4Ye06cF8BmtL1Tq96jjHiR+zlvd2j8vByWOHsfKhY2mT5qHj6HMo6HMx66xMJhi/UpimgpQVegLe8ppBI4cWpJu+nrPKX6dFqSpzM3QtMqVVxPL3PI21SYPY5mrNOi0HPU5l+ZZoSfwrcAr/9J/KD8HeO19RSlsqup+GPehCWD4V1v4Uva9sh4yO9E5QXi0oHqzWm8qxQ7P8LYth++roVN96GLrG345Q/RJ3JVNoWMdohYCUp/0Fxz4Fxz+vLk7veNF09Y/wr55qKl6JaiWhTbuXtnMeZEhgNj31NVg9T4q23bj0JxhwLruTFt6XEnJ263r3d+E+Ye46/jZOfzHacyy/1Eu8y2R8/Co2l6jj2/Oq3sSZpwb+rEsfDsA5VW8DhMryNdi6RK3ghGq91MLtUtbtfwMN6iPJsfs5y52C7U5B63pkzTt1Ha6cpT4A570FrmTwFkfT7XRHdFpAbVO5AMoLYMMsOP45+ORqsIMQl6amnLXo1/AN3fKHypi55IdGvb69jTtOHVyUWU4Gbv0QPecENZozFKQxDY1PrIMZZS3giLLfSUnuz/nFf6MSN+20PMxAOayYRjCYUmPdBWU+FeatHvAxXapWP1yXn15LF/9w6Z9uRN/gDrA63N0lt21bDqp6jnuS+tOvGZ5/bdZ4Vvl7MRDI3DqDVuWzeCcwmo76JrLj1YeYw/Jyt6mumLpTclhPFt0b8RxG0IvPk4PDVhlN8yb/HWfe7/S4/nMA8oqrML2FkKhKgVb2uIqju/21/em94Gg62xs44S+tRTSl0868sNbbM+0CBunLmeduXNDom1aXYfjLqaULW4ztrcYweVMOV/zvH5SRw93lD5D1VgHbSMHjaEl5SsObH1eXYJeQoW9im0cClnuDB8yXeSV4JCPcm4iMJLD8YDjw6urq+2KrHa+2uod7u7WBr8GdoN53rin9JwPMmdiuOi5u1aJ96RwOMr8HwOfzYQUCmJqF6WrYBZV+fftz5eeX8I7zPr7b9h3QgdsynuSM1I7sWCT3i9WdoXroxEOXLN963V3MUODWDxeQmeDiVrst72T2ACDNKuBux0y2XL+FBNNmybLf6/9sK9lE/OJQY/XbNsER1TIbx90Fh9ysyvpXfqd6b/72KlzwP3V/OGiU3rlm9kdmN7jw23r7GVUXDvbE70LQqE/rlBrrEbsgHOTpV0tZ4ub5qr9VyQYVVAIoXkcwIYeE7VVU2U70gedEj7NXfhvN2t9ddF1VZohGCQ8xaEiT+cJyH3EOg2GOPzms7GPWpQ/HXbIKn1f1NPJ7qwjaGnGo77toG2FbAmz8TQ0T2vFiu2ZAqwG79wXt5eQyx34u8ZdHsSf8s+4F5r+rItvX/QGDQ+Uoq9SBFIfeCqNuhDZD6y5nCmcpJbeJRl7jQlN0apvkVZf4TPXGvejDhj9mLxS+kqT7SnAGy1WN9KBJkQ8sj9OgFfmM0hdgGiYT/W/zuvNhciigzA5d4SxaF8k0qq7CF+TGnj/AMf+O3hj+veimmopWV3DwnI/V/Yk5kNRaUmB3kcPQee7SIxjf56+NV91VVYltWe1QUwjV2HsncRMeYJC+HJemMopMLcgppgq+OrM6c6T3QTZsK+b3rycz+/OX63+Cu5PpULUY25WEGZoa0WL5W/QoUVdoZz59Ac898jf6BP/AlaKuiP379H70bfnXsjV6eLaTbDQuI0rsHVr6VON0IzG7UY8bcOxVjL9s5/227PgsVtMakyAfBUfgDh3QbdBb8rB+IXa1HhSNEedRQa6Ctoft0uPF7nWi8SP3my/zT/PCyMUq3fKD6aJYS2aNlU0HbTNtrA2kpWfC3cWRSX4V2Sp7uTE9CE2POk75v+AIyhLa4wsGedR/KmZc8k4eqWQkuLj7RPW8nkARAG0dxfiLN9dY9qNgdFiB7pJyyIboU/YzHTZ/zgnBqcShMsrMUNZsdpKbeE8c3fsPr38l1UtPnfHq+Kc6h1v1ptm6RB1DWdU+gzwZcNXvcO4UGH5t7ON0o/aM+TqEgz1xzr92nd6UzLS/bu5bUFSzWX1EeJJ06Ras1A4kaJWkaOVocalw6O1qSvHXd8LPT++Z7RX1So13kpnoolVK/dn/hq7hDVh4nAbexHZka0X44jKoMhLweVXWUasVkzE0mw12FnOtTvx93UTV2ywuDXJH1FzpXYWxPX4PAPIOtJ/T/BVoG3+ve4Gv/w5vngDLvogGfY4ONXrTdHAnw6Qvwajjwy6tvUrVzB0BPY5TV2WyukFWj9r/yOoS7o1UUfvUm32FK7UFxwUeotBOoDS+HSS1gqzukKmaCsc5DPrrK8jSijBMk6ANg/VljDXmUkAypYkdIbk1QcuqtUY3wWWq5mvX/QE9jlf/jn1K/e50hyo3rI1tqVJCNHU1xQrWvpzYqUG5aZHme3tal7xPuTjvbvVNoArLcNGqXJX4uNyhiX/O6NWQjEQ3V3u+ZuPbVzNgxmUMml13486gX31wJlBBRVJ7xvkewbZtlju6stJqQcDvZ9i2/+Mex+v8NzAOR+ddO1mvzaFXPMfga97a+YJir5OY2YYy243dRFfccrZM5/Giq+mg53GD4wPi7Upu91/Ana6b+LUsm4zsXcs0CgeNkgL79mfO/sLULJxakLNKX4UHWrJ13TJ0y49mOAl2mcBbwbF8Z/Vj9Pb/q/HYoWffw7KTG9cTzZWQygY7gw+DIwlWFOC1dJ4JHo/T0/CMOWfoPVcL9Tc5teR1Wq39iHU/vgW2uvBj2zYPO/5Dha3el72pnRq1nQeqtukexq5+lHuMV8hOUj+7550TWWU1onwnlFFtXV5Pv7UB58KFX4eCRqEG+r+9qkrVXEnwnzGqf8lfEO5FlLALPY3Ebjbj37BtmWrVcHeyamgc7qOWmot93LPqa28JtB/FO8FDmRI8WJUi6ka0JDFchSGa1T9P6cvs28fVCKgOyk0lxaMyT/91at/IlDWP02CrnqUWcnjwmwl4fX7WF1Ywf+V63gyMZbOdRqek0D7x4iF1t2c5AEnQaD/nKFyO9sszdS9gutWIR9tSWTEAqaEsip+fgk+ugl+er/PhgGoKqGlw4otw1W/QaiBcPhMGntfwDdU0uOhbOOiChj9mL5TgMpkfaMuPf24jsXwtrIo9kNU0DT/qAMLM6UGVX6VWHtevFU78JJauhKSWBCy71hrdRLepDka9JXDKa2r6neFUBz4THoO+p9V4DABvnghf3AitD4KjHlMBQbHPMTVI8W9l/dyvKDSz2ebpTFJrFSh0ONQHmxk6gRla9RROU+fgzln4ijbxh1V/dtS2UMP1K3xXs7X1UVTaboJBi4cSb2Os759sy1uHZauD53PMb2id5qlvdY0Sn5JBQmrWbluf2HNSctqRoFVhltXMsNgddMPEsKPT0PJI4xerO30qZrHIPJucuJpZmQ1ar2lSThzdc1vurk0Vu8ElxhQAigu38XTG31nR/gyOOvZUjrv0AUptD3YtpfKmadK1V+OClq7UHHQsXnT8C3PrYvyl23jbcR9OveGTIN1ZqhxcM9V7b9DhQd++irbTLmd7sSo18fqDBG2ND4KjyLeTqcxueIbKgSyrVUcSqaCAZDwJKvurk7WKNlp+w1eiaeRdugwy6ikjM11qGq3pjk5X++xa+Ow6eGoAlOWpScJ/QTjTKDNx13pJ7kpZm6iD6VTZI6Gmx+gG9D5Ffd1uuLrgDdgnv4o5eBIfBA/hXuuC6IXtcInSXwwkit2jruzS9y89mJ4tVdC4bZoncqHX7TA4YZT6rLAdHiZ3f5a5CSPZXuEjgUoOM+bwo9WbxLLV0ZXpEuwNkzPH/Zxl7uTEzhEHVcXq/46HxtbU6g41Lv7Hxxr2ZNPuhTeOg7J8FcHfMKdxG9tqYDTbaR8VjmwD/D7uHTjklhrLBDBYYrVF63pEZMDr4Pbp9M/NYk6/+yCzO4GgTXwtqcwJLhMqC+HZoSoItOE3+OEx6NWARsRbFquI+eCL5ANvH6UZDjr7l9FmyinMTp3AvFan06VbX4r+lh/58DRMkw+DI+igq5N43emhXXAtc61ODOW/da47r8wit2oyS+y2ZGhF/Oa+DL+3goEVP/GL6woKCvLJIzWyvOcvptqL/UNSchrz4kfSrmXTNPDUDQd6taDRod5/Mc11E/dpz1GqxRMXv+ulkfF355HR8sBKL99bvRMYHfO9tzSfDO964i01uTXFv4XTzOnY5u4prY5v2Z3frS7Ea15afX0pG58/gWHGHzjNhl9VTkxK5lzfzeQlqS5GtsODq6oAr+0gYKhyCa+3EkOzOdf8mkytmPTKNbtl+/d3rniV8bXdiPYc6xL8k3KaqLS+y2FwwVR1UU4zoO/p0fL/2npFNkL4hDVrF4NG14zrzKFdM//SNogQw6mmXn10ifo+6FMtOm5cAaNuRHsgR+0D8RkYlQVMc97AY66Xoo8PlzxK9sleydCj5zbh7KJEd/R35XGatO/YlWt8l7MluR/t/Sv58bd5vDB9JeOM34mnkguNL6IrHHKpKlMVgASN9ntlQ67HbldPmVjuSPUBWduo+8KV6v+GljLNeU01igtPGKiqOUVkf+cyDTIS1IHGgBFHgrNm0K7QTqK7vg5W/8iYy56kuMtJ4E7CNHVWtj4edJ2gZZMaX/NDKcFtRqPes19SH1wFf8K0f9S/YWd/WPsEPbFP0aud0HQomE7H8rkApHiiPcc0TePj4HDuNV9Tj3F5aGnnc7gxmxxiR6ZXp63+gbmui/nWdSOZVeoqy5a3LubeqofI0bazSc/h3YAaUzqvx99290sT+yhN1+l302fEJ6XufOFdoJtOTNtHga1OIk81pkfuK9ZTmuQ5xZ73eOBkLvRFp/N5i/M5r/AJWheo6aGepBQA/Obu6QmUmeDiaENN1knVyuhrLcFrOyKT+Boi3mHwuOPZSMaw35WOFqjEpfnZ/OdczrntQXyVZbGPsUp3y/bv75zJ2Xwf7MMC96DIbau1NnwdbKJMrYpCWDMDKrercqX576qsktvzYMLOe6/VJ1yetquZRheP6sirEwf/pW0QIR0OhT+mwMbQRe3ta+GTK9X5S1wqmhUg4dd/oj3WGSqL6Khv5nh7WvTxzni4/FfoU0dWv2hW/dqkRL4OB5CS4kysUJ9Yj9PAbVVykL6c1SnDGLnhRYYGZrN48VxaaQW8zjF01KtlTW/9Q1XjCECCRvs9WzfBVc+V2AmPweW/QKexNe/zqzG0kbTMndF2aISd1bPhG7of2dkY3bl2Z74J9of8pXRN8pM84R/Q6yTMULAIIGBZJNYyCSbBZcamSoYno21bVv9GdRob6ask9l3bW4zkSt9VbLOT6F40nXblC2td7hbzHTrpmwAoajmaWwIXkaGV8LZ9a53rDpYXkKqpExyXR5UD5G6KXnF55u0pjDFUkCoQJxOnxJ5Rld2fm30Xkq6VMtb7KPc5Xo3cV6n/tQbsYu9xrvkVqVopkwNjmKA/xx9pYzEtH3qo3DY+UR1XrG11zG55Pl2PzbadHuyLr5EDhXVDJ10rJSdvOgCLOkziKftUAApWz+NJx9OUWk5u9l/Ed2mn4rcNNEf9DVuFEpeYxnn+W/g4fVLktpHBXyNDHna7LYvg3bPgkfZqWMjy/6mLco64v5wB3yoljgV3H1bnWHCxB439O2T3jn7vK1PVFs74SA+shHn/QQv6IKUtE3038WLGDsdNWd0gScqa90b/nTSYeXeOB8AXVOdTiW4HwVCPOTVlzeYc8xtOTvoDv+bAhZ9kVI+q7o4tsStc/QO8d+4e2/69nQSN9nPeDodhj/l73QvMf1ddWYmr5Spxb3XwQ0On04Q/WE2nKnNLatG4jd1P6Dsp/fruok6MM+aqn9esF+HVI6FgJaauRcZHBi0b06i5niS3IzZoFJ6CJjW3BwTNlch55pc8GzgOI+hF23EEaEhAczBHU0FbhyeJUtuDzzYjE9FqfUxltDTVlVBzgtDHrjvpq69is51GcfbQv/hKhGgY0zDI1raz0mrB4YeMitz+i9WdRxJubsYtE7vTFeYnPOp4kaNue5f+uRkES/Mx7ECkyXS4Z1uGUVbfahply8RfedSvjnOeDx7D/folu7QeV7w62cyy8iiu8FNie3BX5ZOqlVFUVMRnxjiGXfocDi1IoJGBqQOVx6Gzxn0mYyqmRm4rP/oFpvZ/rmmeUDfVsTBAtwlq4Eh9DbQbKckt5Ux7hTmvw5bQxba4VBU08paCOwl0A+s4tX/ZhhPiUmk39ARGnfzXelqJPcfjNCOZ91V+dbwb7zSwQkEjTdMilTVxJauwDRcdtM1Mcd3JD8HezM06Xq3oqGptWaQUMUKCRvs5Ky490titVos/glcOg3W/1LwvPA2noUGjQZOgvlK4A0RtwZ7q2jtDJ+fhzKyitbD2Z0xDIxDJNLJrDT4luE1V8nbTKjj4amjZH8beFV2X2K9lbvuFQfpyMrRi9KBXXQWtRcbV39HpxunqMdvn8oLzcV7WTsChBbGCtQeOglXRsom4pHQ6V73BTf6LmWV1ZbYVbST6v+BgrISm6V8jxI7ii//kMccLVLnS+NsR3dhkpzE5cCiX+K5j3e6LH4hmVmAnkm8nkeJxckzRf+m2/m3K8KBXy5T+rO9zHDJ8VD1raZzsdt1UqTjwsuNRpjsav+6CyxYx6LgrAGhbMpd/OZ4jSatA84V2zjUz+FK/CpfD4EdtIC1z62nKLCIcpjqm6VU1N3LbgIFDOOK4M5vmCXUTEkPZI7NeUCUppZua5rlE81n+JXhCmdKV26NNsV2hqYl9QhfL0zqAYXL3sT3plpPUPNsq/pLwRXhN07CqzzeITAPX6NBjAGeZqvywAjfjDh0HaR2h/9nR5eWifIQEjQ504UyV2k4+k1uDMxG+vqth6xp1E0z8fPdt2z6qQ8ZOei6E34DCU+cANC2mPC1o2XRrUXP0b2Rk66a5cOhtKq3WtmDQhbtr88VeLNzT6HLzExY5elKR3LnW5VqmJ5EcH7pC744n307iE+dRlNge/H5vrY9ZlDqOFwITeCpwPHEZ7eiibaCXtpofg715IaBKQn61ujFQX05W/s9N8OqEqMlwqP3Yb6rgwYfBkXxv9eVz1208568ni1bsUyovmYX3EpXZYZlxaL4Kro27n/JW0QtRBw0bg9vlrGsVu6TfqbcBEK95ecFXd/luXdKz20T6IBmueNro+ay0WrDGoZonB7avx8BG0zRG3vUtaalpu2/j93Nlthv/QXvo2MaZEBskMt0w9bY989xizzGdMPxqGH4N9DgOPOlw1vvQSZU0oelU9DgN+8z3mnc7xV/WMiV6XhvONIqRkIURmvJd2OM8Rpx5G30658LVv6tz4qFXqKyk2ipxDlASPjvQhRtgO+qYspbVHTbM2nPbsx94+swBFFf6615AN9WbUKsBsOq70I0ahq7hD0YzjQ7umMEtR3Sj/a3RvjLh0a28dRK0HQZHPw4/PQGjbmyaFyP2Ku7EaC+hj/TDOTen104f43DHk6mVcKn9Hn28L7FYc1JbUVt+wMMbgZMYoi/lcpfJ5y51wFxix5GkVQJQabsYYixlkSGZbWLP0A0HPtvgpZx7eBoYdtET9PYGaT35iebeNLEbtW4Z7RFiO+LRK/MZ6J1NXLAt0HQ91Fr3PgT+T33dzV71l9ZlutWFnvl2R35zDGCYlYVWuIISM40Ds1j/rzHv2MSwPdUHKLsHDDgPNs9XE7V6HAdF6/bMc4s9Jz4Lvr5Tfd12GLx4CLQZAudHL3iX970QNzJheF/396N7MLRDOkBMJQegmtu3Gx75G0/rfwx03qGq5ogH1D8RIZlGB7r2oXRsRx1jTM+dokqhRIPFu8yYCHcNnjSVFrtmBoy8AYZdCYnZmIZG0KrW00jXImPUk+McvHXhENqlVwvurZupAlC+Uph2b1O+JLGXyGwbbWZ+ZukrZJTupAE64HCpzLdO9lpG6/PweytrXW7civuY7HyA15yPxIwtDQeMAD5hJACGY9emwAjRWKbDiVMLkqGXADCwXRqHdMnks6xL+F/uLc28daIpWO4U/Bbc6n+a+PL1e+x5XVo9F3sawEjKAuBE4yeGbn6LD4Mj2ejzUO6UwQG7Yo82jq7cDr+/DpvnqYlJ0x+EQO1ZuWIf1uUw9b/phoMuUF+v/zVmQlbm2+PRPrt2j2+a2L2yk9ycPbQdAAk7DhYaNAlS26kWH/GZkL+05gqCfnh+JKyXxIkwCRod6PqeDjf+CUmtar/f6YH49D27Tfu75NbQejCU5cG2Fep30GkcZkymkYVZ7cS90hdkeKeMSBApIlzqFm7gKPZrySlp9Kn6D8W2h8MC35MQ2LbTxxipbbjHfw5oBq85HyFQEd1XNhZV8tuaQgDcVdtor+fVuZ43AuM5/ahxap0SNBJ7SmouTweO48jtb8bcfPTlj3Dk+Y0vJxJ7vz87nMtraddi4t8z7zV3F3OF72p+tzr9tfW06Ms//ScD0LtsBh31TbxqHcVXba/bDRspmlRZfvTrQRfCMf+GMXc03/aIptFpHLiT4cJvILdaD1bXDu0gyuo+FhL7nkR3HYVVugE3LFMX73ek6ZC3AOa81qTbti+RoNGB7o8p8OsL6o9D7BkVharkTzNg0Qfw4mjIW4RpVOtpFIxOT+vbOpkJfXZMbg8Fj8JlhdKo7YCx4KFTOdh+lQI7CdNZR1lpNU5Tp622lYCmrrQEfOrq6a+rCnj5P09x1wtvAxDvL2Rum/OYph9cYx0Vtotv7EG0z1V9OrxJ7XfXyxGiXg6CHKwvxtJ3by8bsfdKtYtoVzoXhx3A4dwzAepS4vhn4JS/tI54KikjjllWV+KDJRxrzGRg8Te4UqQ4ba8Xnv6bO1IFEAaeD2nyObff8VeqXqDOeEhqCSe8oG6vFjQqGfo37CMfbaYNFE3hstEdGdstq/Y7dSPaX3bH2wE6H9Z0G7aPkUjBgW7lt/DjY1BbkzDRNAwHJLaAlv3U91YAtizC1KM1t37Lxgg115xy5QgeP61f7DquX6KuhCVmqwi5BI0OKIfEraS1lo/pqqOstBonXiaaX/JV/AR8tkkw4KPC68f76nHcWPYYn7tu4/gnvyXBKqFF70MZe+f/ABhY9Rwneu9mZrAH7wdHcag2h7SsVnwWHIIvUPsENiF2N0ewnAH6CjS5sHHAaBdYzfkF/2Sp3RbDnbDzB+wGt5tv4SDwl9aRgJe7HP8lg2ISbVVOeaf+CkO3f7o7NlE0pfBY7TU/wk+PN++2iKbjiIPzPoPUUECwRd/Q7dELcBX9Jql+R2K/cepBbXj5/EGNf+Dft0HP43f79uyr5CjsQBeemqbLrrDHuBLhhqWQmkskYyjUCDs8IjLc06hOa36ErhPUVZOAFwZObOqtFnuRq4NvsMjOheQ2O13W6VR/48vNbiylHX4L1m0rZpSxEI+mso7+W3AmF3mvRWs1IPK4LvoGOuqbyNCK2WKn0lIrwDAM2l7yHgO6yhVYsWcYoYmBfnPnWXVi/5DaqgtZwa2c4rsTPbn1HnnOrvoGXnE89pfW4YpX2QqP2Wfxb/+JkdvNxMy/tF6xByTkQLejIasnZHRp7q0RTan9yGhmSVZ3uKuo9kwTIQzHzpc5gEik4EA3/l64eW1zb8WBKyGULqlpOAydSn+QG9+fjy9gxTQjruHDi+A/YyBQBbP/o9JsxQGj0pnKlOBwjOQ6epFV43CoD70LS57mHP0hKjxt+OTZ2FHCiVolB+nLSYiLloK87byfRx0v0lnfyN8c73G48RsAfVqn1OytJUQTMUP775wWZzbzlog9JadtZxxakGccT+K091wzYl37axnX8QnJ/JJ+AnFt+vF/wZF8FhwKgCupjrIIsfcwnXD6W9DrBGg5YOfLi/2HHM8I0SASNDrQmS6IS2nurThwHTRRNVtMzcXQNdZvr+SDORso8wbqzzQCKF4HeigKLtPTDiglOcO5y/FfPP7CnS6rhbIIk6xijtF+Iliymb853gUg306OLPd3x5skUPtktdVjnmXlwY/shi0XonGczjhWWTkk+LY296aIPcTtUsHrI4zZOLQ9Vwrrtf/aVWXdMBh61WucXvo6/+e8m0+DqsQlLkWCRvuMjXNV81shhBAxJGgkRHPaNE9NUmszGFPXsKv1lqo30ygs3MvI+mu9GMS+ZdgZtzJv4AO0yGlYhtmzgWNZb+Zyqf0u9qZ5kduP8j7Iw/7TI9/HJybHPG4zGXw58gPajzqLjoddslu2XYjGMB0Otp74fxxxSM0G7WL/dar37wA4nTvv27Y7PBE4kSnB3bOP2YaTTvomcrRC+lS9iKd1r92yXrEHnPoGXPx9c2+FEELsdSRoJERzWvENvHEsbPwdU9ep8EWvqpr19ZlyJoLhitbbuvZMs1Cxd3A6TPodcwWa0bAG6Ifpv6HrOpV6It6i6CjZI41fKSdakmY6YidUFegZHD52/O7ZaCF20dB+vWidmd7cmyH2oH/ddBlvdvwX7rg908tqgdWB/wuO2i3rsg31Pnq9+QEvOJ4gLSFut6xX7AGGqUrVhBBCxJCRS0I0p3At9fbVmEbfmKCRYdSTaXTFL1C4So2E7HwYpLRr4g0V+7JO+iame84jO7CRYOmWyO23mZNZd+pX/HNyOeONOfSp9piDqp5jeJds/r3nN1cIcYBrnRbP2edM2mPPd7bxDQvsDrtlXUFTBYmStXKGGX+A09gt6xVCCCGaS5NlGhUWFnLWWWeRlJRESkoKkyZNoqysrN7HVFVVccUVV5Cenk5CQgInnXQSW7ZsiVlm3bp1TJgwAY/HQ1ZWFjfddBOBQLQ0Z/PmzZx55pl06dIFXde59tprm+LlCbGbRKenmbpGpS+6L9fb02jltxCXpr72ZEDHQ5tuE8U+b42VTaUrk9UJ/cgPJrLCUmVt681cuvQcwOGX/4uys/8X85gpt5zAA2cd0hybK4QQe9QYYx7Xmh/ulnV91+YqXgocyZcJx/Oj3Xe3rFMIIYRoTk2WaXTWWWexefNmvv76a/x+PxMnTuTiiy9m8uTJdT7muuuu4/PPP+f9998nOTmZK6+8khNPPJEZM2YAEAwGmTBhAjk5Ofz8889s3ryZc889F4fDwQMPPACA1+slMzOTO+64g8cff7ypXp4Qu0da9MqmoWuxmUb1BY0+uUr9f3cxzJ8Mw69uqi0U+4FcfQsDK2fwaatrKfMGuSHYj6mnp5OR1RaAXq2Sgdh+Rq1SpKRCCHFg+D5uPIH4bMbuhnVpxet5N3gon117IbpMZhJCCLEfaJKg0ZIlS5g6dSqzZ8/moIMOAuCpp57iqKOO4rHHHqNly5rNW4uLi3n55ZeZPHkyY8aMAeDVV1+le/fu/PLLLwwdOpSvvvqKP/74g2+++Ybs7Gz69evHP/7xD26++WbuvvtunE4nubm5/PvfqqDilVdeaYqXJ8Tu0/N4qPgXZHbDLNZ26GnUiIPN9bMgq/vu3z6xX5g98hW69z+E1V9MoWx7HvFGOR16PNDcmyWEEHuFQ27+YLeta+zG57jNNR1MGR4ghBBi/9AkQaOZM2eSkpISCRgBjBs3Dl3X+fXXXznhhBNqPGbOnDn4/X7GjRsXua1bt260bduWmTNnMnToUGbOnEnv3r3Jzs6OLHP44Ydz2WWXsXjxYvr377/L2+z1evF6vZHvS0pKALAsC8uydnm9zcmyLGzb3me3/4Cw9mdwJUFmN4xV66ioVp6ma9T5uwvXlVqWhdZyAHaHQ2E3/J5ln9k/DTxUved2KJ9Lm+3TyNa2YFn37ZZ1yz4jGkv2GdFY+9I+k+5dD9T9+S32jH1pnxF7B9lnRGPtD/tMQ7e9SYJGeXl5ZGVlxT6RaZKWlkZeXl6dj3E6naSkpMTcnp2dHXlMXl5eTMAofH/4vr/iwQcf5J577qlxe35+PlVVVX9p3c3FsiyKi4uxbRu9vklcotl4lk0n6ZdHKbDiqShvi2VH7yss2EaFWfvvLT2tC2bxWrZu3QrHvg1eYOvWv7w9ss/s33yWTmtrE2iQtxv2F5B9RjSe7DOisfalfeaPvnfww5qFHLWb3mPFrtmX9hmxd5B9RjTW/rDPlJaWNmi5RgWNbrnlFh5++OF6l1myZEljVrnXuPXWW7n++usj35eUlNCmTRsyMzNJSkpqxi3bdZZloWkamZmZ++yOvN9LSAQg1eHHFRcfc1eL7CwcRh2/t/M+xvZVkJWeVfv9u0j2mf3b6riEyNc7BvZ3lewzorFknxGNtS/tM0cffTxwfDNvhdiX9hmxd5B9RjTW/rDPuN3uBi3XqKDRDTfcwPnnn1/vMh06dCAnJ0dlQFQTCAQoLCwkJyen1sfl5OTg8/koKiqKyTbasmVL5DE5OTnMmjUr5nHh6Wp1rbehXC4XLperxu26ru+zOwGApmn7/GvYr4V+L7qu0zUnNjjpMAz0uvoaJbdqsk2SfWb/VZTSC8vW0LXde0VE9hnRWLLPiMaSfUY0luwzorFknxGNta/vMw3d7kYFjTIzM8nMzNzpcsOGDaOoqIg5c+YwcOBAAL799lssy2LIkCG1PmbgwIE4HA6mTZvGSSedBMCyZctYt24dw4YNi6z3/vvvZ+vWrZGr5F9//TVJSUn06NGjMS9FiL1Ddq/QFxrDOqbH3FVnwEiIXZSffTBPBU/g8+AQvmrujRFCCCGEEELs9ZokJNa9e3eOOOIILrroImbNmsWMGTO48sorOf300yOT0zZu3Ei3bt0imUPJyclMmjSJ66+/nu+++445c+YwceJEhg0bxtChQwE47LDD6NGjB+eccw7z58/nyy+/5I477uCKK66IyRKaN28e8+bNo6ysjPz8fObNm8cff/zRFC9ViL+m46FwyuuQ0ytyU4tkN2semtCMGyX2V61K5nKN+SFXD0vf+cJCCCGEEEKIA16TNMIGeOutt7jyyisZO3Ysuq5z0kkn8eSTT0bu9/v9LFu2jIqKishtjz/+eGRZr9fL4YcfzrPPPhu53zAMPvvsMy677DKGDRtGfHw85513Hvfee2/Mc1efojZnzhwmT55Mu3btWLNmTVO9XCF2zYpvoGwLpOZGbnLV0fxaiL8q2asGBnTLbd3MWyKEEEIIIYTYFzRZ0CgtLY3JkyfXeX9ubi62bcfc5na7eeaZZ3jmmWfqfFy7du344osv6n3uHdcrxF4rfzl8eStk94TcEQC4TKOZN0rsv1TJY3Jm0/XEEkIIIYQQQuw/JKVBiOakhfoWecsiN+VmeJppY8T+riKgAuqpGS2aeUuEEEIIIYQQ+4ImyzQSQjREKGgUCh79cNOhpCc4m3F7xP5so6c7t/on8aDD0dybIoQQQgghhNgHSNBIiObUZlDoCxU0apsuWUai6Rw3ZgTZud2bezOEEEIIIYQQ+wgJGgnRnFoNhPM/h4wuzb0l4gDgcZqM6Zbd3JshhBBCCCGE2EdITyMhmtPSz2Hlt5CQ1dxbIoQQQgghhBBCxJCgkRDNqWQT/PhPWDW9ubdECCGEEEIIIYSIIUEjIZpTeHqaFWje7RBCCCGEEEIIIXYgQSMh9gpac2+AEEIIIYQQQggRQ4JGQjSn9oeo/zUJGgkhhBBCCCGE2LvI9DQhmlNGZ7hiNiS1aO4tEUIIIYQQQgghYkimkRDNadGH8MMj4PA095YIIYQQQgghhBAxJGgkRHPylsDC92HNT829JUIIIYQQQgghRAwJGgnRrEK9jKSnkRBCCCGEEEKIvYwEjYRoThIsEkIIIYQQQgixl5KgkRDNqdP40BcSPBJCCCGEEEIIsXeR6WlCNKekFnDzGnAmNPeWCCGEEEIIIYQQMSTTSIjmNP8deGk8WIHm3hIhhBBCCCGEECKGBI2EaE5WEAr+hE3zmntLhBBCCCGEEEKIGBI0EqI5aTI9TQghhBBCCCHE3kmCRkI0K22H/4UQQgghhBBCiL2DBI2EaE7dJoDpBt1o7i0RQgghhBBCCCFiyPQ0IZqT6YIbl4M7ubm3RAghhBBCCCGEiCGZRkI0pz+mwENtoaq4ubdECCGEEEIIIYSIIUEjIZpVqJfRtj+bdzOEEEIIIYQQQogdSNBIiOakSSNsIYQQQgghhBB7JwkaCbE3kJiREEIIIYQQQoi9jASNhGhOPY6HnD5qgpoQQgghhBBCCLEXkelpQjQnXxmc/CpkdGruLRFCCCGEEEIIIWJIppEQzWnVd/D0QKgobO4tEUIIIYQQQgghYkjQSIi9QcnG5t4CIYQQQgghhBAihgSNhGhWoQ7YmvwpCiGEEEIIIYTYu8iZqhDNyXSp/zWjebdDCCGEEEIIIYTYgQSNhGhOXY+ETuPBGd/cWyKEEEIIIYQQQsSQoJEQzal0C4y8HlLaNPeWCCGEEEIIIYQQMSRoJERz2jAbXj0SKouae0uEEEIIIYQQQogYEjQSYm/gLW3uLRBCCCGEEEIIIWJI0EiIvYEujbCFEEIIIYQQQuxdJGgkRHMKN8CW6WlCCCGEEEIIIfYyEjQSojnljoC+Z4Ajrrm3RAghhBBCCCGEiCFBIyGaU+lm6HI4uJOae0uEEEIIIYQQQogYEjQSojltmgvvnw/+yubeEiGEEEIIIYQQIoYEjYRoTgGf+t8KNO92CCGEEEIIIYQQO5CgkRDNybbU/9IIWwghhBBCCCHEXkaCRkI0J0+6+l+XoJEQQgghhBBCiL1LkwaNCgsLOeuss0hKSiIlJYVJkyZRVlZW72Oqqqq44oorSE9PJyEhgZNOOoktW7bELLNu3TomTJiAx+MhKyuLm266iUAgWt7z4YcfMn78eDIzM0lKSmLYsGF8+eWXTfIahfhL2g6BkTeCbjb3lgghhBBCCCGEEDGaNGh01llnsXjxYr7++ms+++wzfvjhBy6++OJ6H3Pdddfx6aef8v777/P999+zadMmTjzxxMj9wWCQCRMm4PP5+Pnnn3n99dd57bXXuPPOOyPL/PDDD4wfP54vvviCOXPmcOihh3LMMccwd+7cJnutQuySsq0q20gyjYQQQgghhBBC7GU027btpljxkiVL6NGjB7Nnz+aggw4CYOrUqRx11FFs2LCBli1b1nhMcXExmZmZTJ48mZNPPhmApUuX0r17d2bOnMnQoUP53//+x9FHH82mTZvIzs4G4Pnnn+fmm28mPz8fp9NZ6/b07NmT0047LSa4VJ+SkhKSk5MpLi4mKWnfHIduWRZbt24lKysLXZdKxL3S/Hfho4vh7uLm3hJA9hnReLLPiMaSfUY0luwzorFknxGNJfuMaKz9YZ9paMyjyWpiZs6cSUpKSiRgBDBu3Dh0XefXX3/lhBNOqPGYOXPm4Pf7GTduXOS2bt260bZt20jQaObMmfTu3TsSMAI4/PDDueyyy1i8eDH9+/evsV7LsigtLSUtLa3O7fV6vXi93sj3JSUlkcdaltW4F7+XsCwL27b32e0/IPjK0WGv+R3JPiMaS/YZ0Viyz4jGkn1GNJbsM6KxZJ8RjbU/7DMN3fYmCxrl5eWRlZUV+2SmSVpaGnl5eXU+xul0kpKSEnN7dnZ25DF5eXkxAaPw/eH7avPYY49RVlbGqaeeWuf2Pvjgg9xzzz01bs/Pz6eqqqrOx+3NLMuiuLgY27b32ejn/i6upJhkYOvWrc29KYDsM6LxZJ8RjSX7jGgs2WdEY8k+IxpL9hnRWPvDPlNaWtqg5RodNLrlllt4+OGH611myZIljV1tk5k8eTL33HMPU6ZMqRHEqu7WW2/l+uuvj3xfUlJCmzZtIs2090WWZaFpGpmZmfvsjrzf294ZoN59c0+SfUY0luwzorFknxGNJfuMaCzZZ0RjyT4jGmt/2GfcbneDlmt00OiGG27g/PPPr3eZDh06kJOTUyN7IhAIUFhYSE5OTq2Py8nJwefzUVRUFJNttGXLlshjcnJymDVrVszjwtPVdlzvO++8w4UXXsj7778fU/JWG5fLhcvlqnG7ruv77E4AoGnaPv8a9mu5w+GYJ/eq34/sM6KxZJ8RjSX7jGgs2WdEY8k+IxpL9hnRWPv6PtPQ7W500CgzM5PMzMydLjds2DCKioqYM2cOAwcOBODbb7/FsiyGDBlS62MGDhyIw+Fg2rRpnHTSSQAsW7aMdevWMWzYsMh677///kjTKYCvv/6apKQkevToEVnX22+/zQUXXMA777zDhAkTGvsyhdgzyrZCwZ/NvRVCCCGEEEIIIUQNTRYS6969O0cccQQXXXQRs2bNYsaMGVx55ZWcfvrpkclpGzdupFu3bpHMoeTkZCZNmsT111/Pd999x5w5c5g4cSLDhg1j6NChABx22GH06NGDc845h/nz5/Pll19yxx13cMUVV0QyhSZPnsy5557LP//5T4YMGUJeXh55eXkUF+8dE6qEiFjzI/z8VHNvhRBCCCGEEEIIUUOT5lG99dZbdOvWjbFjx3LUUUcxYsQIXnzxxcj9fr+fZcuWUVFREbnt8ccf5+ijj+akk05i1KhR5OTk8OGHH0buNwyDzz77DMMwGDZsGGeffTbnnnsu9957b2SZF198kUAgwBVXXEGLFi0i/6655pqmfLlCNJ6/srm3QAghhBBCCCGEqFWTTU8DSEtLY/LkyXXen5ubi23bMbe53W6eeeYZnnnmmTof165dO7744os6758+fXqjt1WIZrHD/i+EEEIIIYQQQuwt9s2OTULsL5JbNfcWCCGEEEIIIYQQtZKgkRDNqeMYOHdKc2+FEEIIIYQQQghRgwSNhGhOcanQYXRzb4UQQgghhBBCCFGDBI2EEEIIIYQQQgghRA0SNBJCCCGEEEIIIYQQNUjQSAghhBBCCCGEEELUIEEjIYQQQgghhBBCCFGDBI2EEEIIIYQQQgghRA0SNBJCCCGEEEIIIYQQNUjQSAghhBBCCCGEEELUIEEjIYQQQgghhBBCCFGD2dwbsLeybRuAkpKSZt6SXWdZFqWlpbjdbnRd4oNi52SfEY0l+4xoLNlnRGPJPiMaS/YZ0Viyz4jG2h/2mXCsIxz7qIsEjepQWloKQJs2bZp5S4QQQgghhBBCCCF2v9LSUpKTk+u8X7N3FlY6QFmWxaZNm0hMTETTtObenF1SUlJCmzZtWL9+PUlJSc29OWIfIPuMaCzZZ0RjyT4jGkv2GdFYss+IxpJ9RjTW/rDP2LZNaWkpLVu2rDdbSjKN6qDrOq1bt27uzdgtkpKS9tkdWTQP2WdEY8k+IxpL9hnRWLLPiMaSfUY0luwzorH29X2mvgyjsH2z+E4IIYQQQgghhBBCNCkJGgkhhBBCCCGEEEKIGiRotB9zuVzcdddduFyu5t4UsY+QfUY0luwzorFknxGNJfuMaCzZZ0RjyT4jGutA2mekEbYQQgghhBBCCCGEqEEyjYQQQgghhBBCCCFEDRI0EkIIIYQQQgghhBA1SNBICCGEEEIIIYQQQtQgQSMhhBBCCCGEEEIIUYMEjfYxzzzzDLm5ubjdboYMGcKsWbPqXf7999+nW7duuN1uevfuzRdffBFzv23b3HnnnbRo0YK4uDjGjRvHn3/+2ZQvQexhu3Of8fv93HzzzfTu3Zv4+HhatmzJueeey6ZNm5r6ZYg9aHe/z1R36aWXomkaTzzxxG7eatGcmmKfWbJkCcceeyzJycnEx8czaNAg1q1b11QvQexBu3t/KSsr48orr6R169bExcXRo0cPnn/++aZ8CWIPa8w+s3jxYk466SRyc3Pr/bxp7H4o9i27e5958MEHGTRoEImJiWRlZXH88cezbNmyJnwFYk9riveZsIceeghN07j22mt370bvKbbYZ7zzzju20+m0X3nlFXvx4sX2RRddZKekpNhbtmypdfkZM2bYhmHYjzzyiP3HH3/Yd9xxh+1wOOyFCxdGlnnooYfs5ORk++OPP7bnz59vH3vssXb79u3tysrKPfWyRBPa3ftMUVGRPW7cOPvdd9+1ly5das+cOdMePHiwPXDgwD35skQTaor3mbAPP/zQ7tu3r92yZUv78ccfb+JXIvaUpthnVqxYYaelpdk33XST/fvvv9srVqywp0yZUuc6xb6jKfaXiy66yO7YsaP93Xff2atXr7ZfeOEF2zAMe8qUKXvqZYkm1Nh9ZtasWfaNN95ov/3223ZOTk6tnzeNXafYtzTFPnP44Yfbr776qr1o0SJ73rx59lFHHWW3bdvWLisra+JXI/aEpthnqi+bm5tr9+nTx77mmmua5gU0MQka7UMGD/7/9u4vpKn/jQP427bmrlJKci3RhDJyUWF/xIJ2oaQgJXWhREldFTHpDyVFEUE3GllQoz8EYXdJF2WSF7VsmZUW+AeTIoVEKpyji6zIVDzP76r9vnP65Wuezzk78X7BEM8++/A87M3x8Dg9G8Tn80W+n5iYELfbLVVVVVOuLy0tleLi4qhjubm5sn//fhER0TRNXC6XnD9/PvL8169fJTExUW7fvq2gAzKa3pmZyuvXrwWADAwM6FM0mUpVZj59+iSLFy+Wnp4eycjI4NDoL6IiM2VlZbJ79241BZOpVOTF4/HI2bNno9bk5OTIqVOndKyczDLTzPzTdD9vZrMnxT8VmZksHA4LAGlubp5NqRQnVGXm+/fvsmzZMgkEAuL1ei07NOKfp1nE2NgY2tvbUVBQEDk2Z84cFBQUoLW1dcrXtLa2Rq0HgMLCwsj6/v5+hEKhqDVJSUnIzc2ddk+yDhWZmcrw8DASEhKQnJysS91kHlWZ0TQN5eXlqKyshMfjUVM8mUJFZjRNQ2NjI7KyslBYWIiFCxciNzcX9fX1yvogY6g6x2zcuBENDQ34/PkzRATBYBC9vb3YsmWLmkbIMH+SGTP2pPhh1Ps7PDwMAJg/f75ue5I5VGbG5/OhuLg45ueY1XBoZBFfvnzBxMQEUlNTo46npqYiFApN+ZpQKPSv639/ncmeZB0qMjPZr1+/cPz4cezcuRPz5s3Tp3AyjarMnDt3Dna7HQcPHtS/aDKVisyEw2H8+PED1dXVKCoqwqNHj7B9+3bs2LEDzc3NahohQ6g6x/j9fmRnZyMtLQ0OhwNFRUW4cuUKNm/erH8TZKg/yYwZe1L8MOL91TQNhw8fxqZNm7By5Upd9iTzqMpMXV0dOjo6UFVVNdsSTWc3uwAisqbx8XGUlpZCRHDt2jWzy6E41d7ejkuXLqGjowMJCQlml0MWoGkaAKCkpARHjhwBAKxZswYvX77E9evX4fV6zSyP4pDf70dbWxsaGhqQkZGBZ8+ewefzwe12W/63u0QUf3w+H3p6evD8+XOzS6E49fHjRxw6dAiBQABOp9PscmaNnzSyiJSUFNhsNgwNDUUdHxoagsvlmvI1LpfrX9f//jqTPck6VGTmt98Do4GBAQQCAX7K6C+hIjMtLS0Ih8NIT0+H3W6H3W7HwMAAjh49iiVLlijpg4yjIjMpKSmw2+3Izs6OWrNixQrePc3iVORlZGQEJ0+exMWLF7F161asWrUKFRUVKCsrQ01NjZpGyDB/khkz9qT4ofr9raiowIMHDxAMBpGWljbr/ch8KjLT3t6OcDiMnJycyPVvc3MzLl++DLvdjomJCT1KNwyHRhbhcDiwdu1aNDU1RY5pmoampibk5eVN+Zq8vLyo9QAQCAQi6zMzM+FyuaLWfPv2Da9evZp2T7IOFZkB/j8w6uvrw+PHj7FgwQI1DZDhVGSmvLwc3d3d6OrqijzcbjcqKyvx8OFDdc2QIVRkxuFwYP369TG3Mu7t7UVGRobOHZCRVORlfHwc4+PjmDMn+pLWZrNFPrVG1vUnmTFjT4ofqt5fEUFFRQXu3buHJ0+eIDMzU49yKQ6oyEx+fj7evHkTdf27bt067Nq1C11dXbDZbHqVbwyT/xE3zUBdXZ0kJibKrVu35O3bt7Jv3z5JTk6WUCgkIiLl5eVy4sSJyPoXL16I3W6XmpoaeffunZw5cybmNrXV1dWSnJws9+/fl+7ubikpKZHMzEwZGRkxvD/Sn96ZGRsbk23btklaWpp0dXXJ4OBg5DE6OmpKj6QvFeeZyXj3tL+LiszcvXtX5s6dKzdu3JC+vj7x+/1is9mkpaXF8P5IXyry4vV6xePxSDAYlA8fPkhtba04nU65evWq4f2R/maamdHRUens7JTOzk5ZtGiRHDt2TDo7O6Wvr+8/70nWpiIzBw4ckKSkJHn69GnU9e/Pnz8N74/0pyIzk1n57mkcGlmM3++X9PR0cTgcsmHDBmlra4s85/V6Zc+ePVHr79y5I1lZWeJwOMTj8UhjY2PU85qmyenTpyU1NVUSExMlPz9f3r9/b0QrZBA9M9Pf3y8ApnwEg0GDOiLV9D7PTMah0d9HRWZu3rwpS5cuFafTKatXr5b6+nrVbZBB9M7L4OCg7N27V9xutzidTlm+fLlcuHBBNE0zoh0ywEwyM921itfr/c97kvXpnZnprn9ra2uNa4qUUnGe+ScrD40SREQM+lATERERERERERFZBP+nERERERERERERxeDQiIiIiIiIiIiIYnBoREREREREREREMTg0IiIiIiIiIiKiGBwaERERERERERFRDA6NiIiIiIiIiIgoBodGREREREREREQUg0MjIiIiIiIiIiKKwaERERERERERERHF4NCIiIiIiIiIiIhicGhEREREREREREQxODQiIiIiIiIiIqIY/wMeUzHtPMBS3gAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "import matplotlib.pyplot as plt\n", + "\n", + "n = min(len(full_wav), len(stream_wav))\n", + "a, b = full_wav[:n], stream_wav[:n]\n", + "diff = a - b\n", + "print(f\"length: full={len(full_wav)} stream={len(stream_wav)} (compared first {n})\")\n", + "print(f\"max |diff| = {np.abs(diff).max():.3e} RMS diff = {np.sqrt((diff**2).mean()):.3e}\")\n", + "print(f\"signal RMS = {np.sqrt((a**2).mean()):.3e}\")\n", + "\n", + "t = np.arange(n) / SR\n", + "fig, ax = plt.subplots(3, 1, figsize=(14, 8), sharex=True)\n", + "ax[0].plot(t, a, lw=0.5)\n", + "ax[0].set_title(\"Full one-shot decode\")\n", + "ax[1].plot(t, b, lw=0.5, color=\"tab:orange\")\n", + "ax[1].set_title(f\"Streaming decode (L={LEFT_CONTEXT} + N={CHUNK_FRAMES}/step, R={RIGHT_CONTEXT})\")\n", + "ax[2].plot(t, diff, lw=0.5, color=\"tab:red\")\n", + "ax[2].set_title(\"Difference (full - streaming)\")\n", + "ax[2].set_xlabel(\"time (s)\")\n", + "for a_ in ax:\n", + " a_.grid(alpha=0.3)\n", + "plt.tight_layout()\n", + "plt.show()\n", + "\n", + "# Zoom on a chunk boundary region (~middle) to inspect seams.\n", + "mid = n // 2\n", + "w = min(SR // 2, n - mid)\n", + "plt.figure(figsize=(14, 3))\n", + "plt.plot(np.arange(w) / SR, a[mid : mid + w], lw=0.8, label=\"full\")\n", + "plt.plot(np.arange(w) / SR, b[mid : mid + w], lw=0.8, ls=\"--\", label=\"stream\")\n", + "plt.title(\"Zoom near mid-utterance (look for seams at 3-frame / 120 ms boundaries)\")\n", + "plt.legend()\n", + "plt.grid(alpha=0.3)\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "42acee3a", + "metadata": {}, + "source": [ + "## 6. Listen\n", + "\n", + "Play both and save them as wavs for offline comparison." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "1522184c", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "wrote decode_full.wav and decode_stream.wav\n", + "Full one-shot decode:\n" + ] + }, + { + "data": { + "text/html": [ + "\n", + " \n", + " " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Streaming decode (24 ctx + 3/step):\n" + ] + }, + { + "data": { + "text/html": [ + "\n", + " \n", + " " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "import soundfile as sf\n", + "from IPython.display import Audio, display\n", + "\n", + "sf.write(\"decode_full.wav\", full_wav, SR)\n", + "sf.write(\"decode_stream.wav\", stream_wav, SR)\n", + "print(\"wrote decode_full.wav and decode_stream.wav\")\n", + "\n", + "print(\"Full one-shot decode:\")\n", + "display(Audio(full_wav, rate=SR))\n", + "print(\"Streaming decode (24 ctx + 3/step):\")\n", + "display(Audio(stream_wav, rate=SR))" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "emp", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.13" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}