import{s as Ms,n as hs,o as ys}from"../chunks/scheduler.39a43cde.js";import{S as ms,i as us,g as i,s as n,r,A as ds,h as o,f as s,c as a,j as ge,u as c,x as p,k as fe,y as ws,a as l,v as M,d as h,t as y,w as m}from"../chunks/index.23ec2204.js";import{D as Js,C as u}from"../chunks/DocNotebookDropdown.21a75b74.js";import{H as d}from"../chunks/Heading.c42e7061.js";function js(vt){let w,Ie,Ue,Be,J,Ce,j,Ze,T,kt='<em>Authored by: <a href="https://huggingface.co/andrewrreed" rel="nofollow">Andrew Reed</a></em>',Ge,U,_t="This notebook demonstrates how you can easily transition from OpenAI models to Open LLMs without needing to refactor any existing code.",We,b,Rt='<a href="https://github.com/huggingface/text-generation-inference" rel="nofollow">Text Generation Inference (TGI)</a> now offers a <a href="https://huggingface.co/blog/tgi-messages-api" rel="nofollow">Messages API</a>, making it directly compatible with the OpenAI Chat Completion API. This means that any existing scripts that use OpenAI models (via the OpenAI client library or third-party tools like LangChain or LlamaIndex) can be directly swapped out to use any open LLM running on a TGI endpoint!',ve,g,Et="This allows you to quickly test out and benefit from the numerous advantages offered by open models. Things like:",ke,f,At="<li>Complete control and transparency over models and data</li> <li>No more worrying about rate limits</li> <li>The ability to fully customize systems according to your specific needs</li>",_e,I,Nt="In this notebook, we’ll show you how to:",Re,B,xt='<li><a href="#section_1">Create Inference Endpoint to Deploy a Model with TGI</a></li> <li><a href="#section_2">Query the Inference Endpoint with OpenAI Client Libraries</a></li> <li><a href="#section_3">Integrate the Endpoint with LangChain and LlamaIndex Workflows</a></li>',Ee,C,Vt="<strong>Let’s dive in!</strong>",Ae,Z,Ne,G,Xt="First we need to install dependencies and set an HF API key.",xe,W,Ve,v,Xe,Je,Qe,k,Se,_,Qt='To get started, let’s deploy <a href="https://huggingface.co/NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO" rel="nofollow">Nous-Hermes-2-Mixtral-8x7B-DPO</a>, a fine-tuned Mixtral model, to Inference Endpoints using TGI.',$e,R,St='We can deploy the model in just <a href="https://ui.endpoints.huggingface.co/new?vendor=aws&amp;repository=NousResearch%2FNous-Hermes-2-Mixtral-8x7B-DPO&amp;tgi_max_total_tokens=32000&amp;tgi=true&amp;tgi_max_input_length=1024&amp;task=text-generation&amp;instance_size=2xlarge&amp;tgi_max_batch_prefill_tokens=2048&amp;tgi_max_batch_total_tokens=1024000&amp;no_suggested_compute=true&amp;accelerator=gpu&amp;region=us-east-1" rel="nofollow">a few clicks from the UI</a>, or take advantage of the <code>huggingface_hub</code> Python library to programmatically create and manage Inference Endpoints.',Fe,E,$t='We’ll use the Hub library here by specifing an endpoint name and model repository, along with the task of <code>text-generation</code>. In this example, we use a <code>protected</code> type so access to the deployed model will require a valid Hugging Face token. We also need to configure the hardware requirements like vendor, region, accelerator, instance type, and size. You can check out the list of available resource options <a href="https://api.endpoints.huggingface.cloud/#get-/v2/provider" rel="nofollow">using this API call</a>, and view recommended configurations for select models in the catalog <a href="https://ui.endpoints.huggingface.co/catalog" rel="nofollow">here</a>.',He,A,Ft='<em>Note: You may need to request a quota upgrade by sending an email to <a href="mailto:api-enterprise@huggingface.co">api-enterprise@huggingface.co</a></em>',qe,N,Ye,x,Ht=`running
`,Le,V,qt="It will take a few minutes for our deployment to spin up. We can use the <code>.wait()</code> utility to block the running thread until the endpoint reaches a final “running” state. Once running, we can confirm its status and take it for a spin via the UI Playground:",ze,X,Yt='<img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/blog/messages-api/endpoint-overview.png" alt="IE UI Overview"/>',Pe,Q,Lt="Great, we now have a working endpoint!",Oe,S,zt='<em>Note: When deploying with <code>huggingface_hub</code>, your endpoint will scale-to-zero after 15 minutes of idle time by default to optimize cost during periods of inactivity. Check out <a href="https://huggingface.co/docs/huggingface_hub/guides/inference_endpoints" rel="nofollow">the Hub Python Library documentation</a> to see all the functionality available for managing your endpoint lifecycle.</em>',De,je,Ke,$,et,F,Pt="As mentioned above, since our model is hosted with TGI it now supports a Messages API meaning we can query it directly using the familiar OpenAI client libraries.",tt,H,st,q,Ot='The example below shows how to make this transition using the <a href="https://github.com/openai/openai-python" rel="nofollow">OpenAI Python Library</a>. Simply replace the <code>&lt;ENDPOINT_URL&gt;</code> with your endpoint URL (be sure to include the <code>v1/</code> the suffix) and populate the <code>&lt;HF_API_KEY&gt;</code> field with a valid Hugging Face user token. The <code>&lt;ENDPOINT_URL&gt;</code> can be gathered from Inference Endpoints UI, or from the endpoint object we created above with <code>endpoint.url</code>.',lt,Y,Dt="We can then use the client as usual, passing a list of messages to stream responses from our Inference Endpoint.",nt,L,at,z,Kt=`Open-source software is important due to a number of reasons, including:

1. Collaboration: The collaborative nature of open-source software allows developers from around the world to work together, share their ideas and improve the code. This often results in faster progress and better software.

2. Transparency: With open-source software, the code is publicly available, making it easy to see exactly how the software functions, and allowing users to determine if there are any security vulnerabilities.

3. Customization: Being able to access the code also allows users to customize the software to better suit their needs. This makes open-source software incredibly versatile, as users can tweak it to suit their specific use case.

4. Quality: Open-source software is often developed by large communities of dedicated developers, who work together to improve the software. This results in a higher level of quality than might be found in proprietary software.

5. Cost: Open-source software is often provided free of charge, which makes it accessible to a wider range of users. This can be especially important for organizations with limited budgets for software.

6. Shared Benefit: By sharing the code of open-source software, everyone can benefit from the hard work of the developers. This contributes to the overall advancement of technology, as users and developers work together to improve and build upon the software.

In summary, open-source software provides a collaborative platform that leads to high-quality, customizable, and transparent software, all available at little or no cost, benefiting both individuals and the technology community as a whole.<|im_end|>
`,it,P,es='Behind the scenes, TGI’s Messages API automatically converts the list of messages into the model’s required instruction format using its <a href="https://huggingface.co/docs/transformers/chat_templating" rel="nofollow">chat template</a>.',ot,O,ts="<em>Note: Certain OpenAI features, like function calling, are not compatible with TGI. Currently, the Messages API supports the following chat completion parameters: <code>stream</code>, <code>max_new_tokens</code>, <code>frequency_penalty</code>, <code>logprobs</code>, <code>seed</code>, <code>temperature</code>, and <code>top_p</code>.</em>",pt,D,rt,K,ss='Here’s the same streaming example above, but using the <a href="https://github.com/openai/openai-node" rel="nofollow">OpenAI Javascript/Typescript Library</a>.',ct,ee,Mt,Te,ht,te,yt,se,ls="Now, let’s see how to use this newly created endpoint with popular RAG frameworks like LangChain and LlamaIndex.",mt,le,ut,ne,ns='To use it in <a href="https://python.langchain.com/docs/get_started/introduction" rel="nofollow">LangChain</a>, simply create an instance of <code>ChatOpenAI</code> and pass your <code>&lt;ENDPOINT_URL&gt;</code> and <code>&lt;HF_API_TOKEN&gt;</code> as follows:',dt,ae,wt,ie,as="We’re able to directly leverage the same <code>ChatOpenAI</code> class that we would have used with the OpenAI models. This allows all previous code to work with our endpoint by changing just one line of code.",Jt,oe,is="Let’s now use our Mixtral model in a simple RAG pipeline to answer a question over the contents of a HF blog post.",jt,pe,Tt,re,Ut,ce,os='Similarly, you can also use a TGI endpoint in <a href="https://www.llamaindex.ai/" rel="nofollow">LlamaIndex</a>. We’ll use the <code>OpenAILike</code> class, and instantiate it by configuring some additional arguments (i.e. <code>is_local</code>, <code>is_function_calling_model</code>, <code>is_chat_model</code>, <code>context_window</code>).',bt,Me,ps="<em>Note: that the context window argument should match the value previously set for <code>MAX_TOTAL_TOKENS</code> of your endpoint.</em>",gt,he,ft,ye,rs="We can now use it in a similar RAG pipeline. Keep in mind that the previous choice of <code>MAX_INPUT_LENGTH</code> in your Inference Endpoint will directly influence the number of retrieved chunk (<code>similarity_top_k</code>) the model can process.",It,me,Bt,ue,Ct,de,cs="After you are done with your endpoint, you can either pause or delete it. This step can be completed via the UI, or programmatically like follows.",Zt,we,Gt,be,Wt;return J=new Js({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/cookbook/blob/main/notebooks/en/tgi_messages_api_demo.ipynb"}]}}),j=new d({props:{title:"Migrating from OpenAI to Open LLMs Using TGI’s Messages API",local:"migrating-from-openai-to-open-llms-using-tgis-messages-api",headingTag:"h1"}}),Z=new d({props:{title:"Setup",local:"setup",headingTag:"h2"}}),W=new u({props:{code:"IXBpcCUyMGluc3RhbGwlMjAtLXVwZ3JhZGUlMjAtcSUyMGh1Z2dpbmdmYWNlX2h1YiUyMGxhbmdjaGFpbiUyMGxhbmdjaGFpbi1jb21tdW5pdHklMjBsYW5nY2hhaW5odWIlMjBsYW5nY2hhaW4tb3BlbmFpJTIwbGxhbWEtaW5kZXglMjBjaHJvbWFkYiUyMGJzNCUyMHNlbnRlbmNlX3RyYW5zZm9ybWVycyUyMHRvcmNo",highlighted:"!pip install --upgrade -q huggingface_hub langchain langchain-community langchainhub langchain-openai llama-index chromadb bs4 sentence_transformers torch",wrap:!1}}),v=new u({props:{code:"aW1wb3J0JTIwb3MlMEFpbXBvcnQlMjBnZXRwYXNzJTBBJTBBJTIzJTIwZW50ZXIlMjBBUEklMjBrZXklMEFvcy5lbnZpcm9uJTVCJTIySFVHR0lOR0ZBQ0VIVUJfQVBJX1RPS0VOJTIyJTVEJTIwJTNEJTIwSEZfQVBJX0tFWSUyMCUzRCUyMGdldHBhc3MuZ2V0cGFzcygp",highlighted:`<span class="hljs-keyword">import</span> os
<span class="hljs-keyword">import</span> getpass

<span class="hljs-comment"># enter API key</span>
os.environ[<span class="hljs-string">&quot;HUGGINGFACEHUB_API_TOKEN&quot;</span>] = HF_API_KEY = getpass.getpass()`,wrap:!1}}),k=new d({props:{title:"1. Create an Inference Endpoint",local:"1-create-an-inference-endpoint",headingTag:"h2"}}),N=new u({props:{code:"ZnJvbSUyMGh1Z2dpbmdmYWNlX2h1YiUyMGltcG9ydCUyMGNyZWF0ZV9pbmZlcmVuY2VfZW5kcG9pbnQlMEElMEFlbmRwb2ludCUyMCUzRCUyMGNyZWF0ZV9pbmZlcmVuY2VfZW5kcG9pbnQoJTBBJTIwJTIwJTIwJTIwJTIybm91cy1oZXJtZXMtMi1taXh0cmFsLTh4N2ItZGVtbyUyMiUyQyUwQSUyMCUyMCUyMCUyMHJlcG9zaXRvcnklM0QlMjJOb3VzUmVzZWFyY2glMkZOb3VzLUhlcm1lcy0yLU1peHRyYWwtOHg3Qi1EUE8lMjIlMkMlMEElMjAlMjAlMjAlMjBmcmFtZXdvcmslM0QlMjJweXRvcmNoJTIyJTJDJTBBJTIwJTIwJTIwJTIwdGFzayUzRCUyMnRleHQtZ2VuZXJhdGlvbiUyMiUyQyUwQSUyMCUyMCUyMCUyMGFjY2VsZXJhdG9yJTNEJTIyZ3B1JTIyJTJDJTBBJTIwJTIwJTIwJTIwdmVuZG9yJTNEJTIyYXdzJTIyJTJDJTBBJTIwJTIwJTIwJTIwcmVnaW9uJTNEJTIydXMtZWFzdC0xJTIyJTJDJTBBJTIwJTIwJTIwJTIwdHlwZSUzRCUyMnByb3RlY3RlZCUyMiUyQyUwQSUyMCUyMCUyMCUyMGluc3RhbmNlX3R5cGUlM0QlMjJwNGRlJTIyJTJDJTBBJTIwJTIwJTIwJTIwaW5zdGFuY2Vfc2l6ZSUzRCUyMjJ4bGFyZ2UlMjIlMkMlMEElMjAlMjAlMjAlMjBjdXN0b21faW1hZ2UlM0QlN0IlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJoZWFsdGhfcm91dGUlMjIlM0ElMjAlMjIlMkZoZWFsdGglMjIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJlbnYlMjIlM0ElMjAlN0IlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJNQVhfSU5QVVRfTEVOR1RIJTIyJTNBJTIwJTIyNDA5NiUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMk1BWF9CQVRDSF9QUkVGSUxMX1RPS0VOUyUyMiUzQSUyMCUyMjQwOTYlMjIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJNQVhfVE9UQUxfVE9LRU5TJTIyJTNBJTIwJTIyMzIwMDAlMjIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJNQVhfQkFUQ0hfVE9UQUxfVE9LRU5TJTIyJTNBJTIwJTIyMTAyNDAwMCUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMk1PREVMX0lEJTIyJTNBJTIwJTIyJTJGcmVwb3NpdG9yeSUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCU3RCUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnVybCUyMiUzQSUyMCUyMmdoY3IuaW8lMkZodWdnaW5nZmFjZSUyRnRleHQtZ2VuZXJhdGlvbi1pbmZlcmVuY2UlM0FzaGEtMTczNDU0MCUyMiUyQyUyMCUyMCUyMyUyMG11c3QlMjBiZSUyMCUzRSUzRCUyMDEuNC4wJTBBJTIwJTIwJTIwJTIwJTdEJTJDJTBBKSUwQSUwQWVuZHBvaW50LndhaXQoKSUwQXByaW50KGVuZHBvaW50LnN0YXR1cyk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> create_inference_endpoint

<span class="hljs-meta">&gt;&gt;&gt; </span>endpoint = create_inference_endpoint(
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;nous-hermes-2-mixtral-8x7b-demo&quot;</span>,
<span class="hljs-meta">... </span>    repository=<span class="hljs-string">&quot;NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO&quot;</span>,
<span class="hljs-meta">... </span>    framework=<span class="hljs-string">&quot;pytorch&quot;</span>,
<span class="hljs-meta">... </span>    task=<span class="hljs-string">&quot;text-generation&quot;</span>,
<span class="hljs-meta">... </span>    accelerator=<span class="hljs-string">&quot;gpu&quot;</span>,
<span class="hljs-meta">... </span>    vendor=<span class="hljs-string">&quot;aws&quot;</span>,
<span class="hljs-meta">... </span>    region=<span class="hljs-string">&quot;us-east-1&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-built_in">type</span>=<span class="hljs-string">&quot;protected&quot;</span>,
<span class="hljs-meta">... </span>    instance_type=<span class="hljs-string">&quot;p4de&quot;</span>,
<span class="hljs-meta">... </span>    instance_size=<span class="hljs-string">&quot;2xlarge&quot;</span>,
<span class="hljs-meta">... </span>    custom_image={
<span class="hljs-meta">... </span>        <span class="hljs-string">&quot;health_route&quot;</span>: <span class="hljs-string">&quot;/health&quot;</span>,
<span class="hljs-meta">... </span>        <span class="hljs-string">&quot;env&quot;</span>: {
<span class="hljs-meta">... </span>            <span class="hljs-string">&quot;MAX_INPUT_LENGTH&quot;</span>: <span class="hljs-string">&quot;4096&quot;</span>,
<span class="hljs-meta">... </span>            <span class="hljs-string">&quot;MAX_BATCH_PREFILL_TOKENS&quot;</span>: <span class="hljs-string">&quot;4096&quot;</span>,
<span class="hljs-meta">... </span>            <span class="hljs-string">&quot;MAX_TOTAL_TOKENS&quot;</span>: <span class="hljs-string">&quot;32000&quot;</span>,
<span class="hljs-meta">... </span>            <span class="hljs-string">&quot;MAX_BATCH_TOTAL_TOKENS&quot;</span>: <span class="hljs-string">&quot;1024000&quot;</span>,
<span class="hljs-meta">... </span>            <span class="hljs-string">&quot;MODEL_ID&quot;</span>: <span class="hljs-string">&quot;/repository&quot;</span>,
<span class="hljs-meta">... </span>        },
<span class="hljs-meta">... </span>        <span class="hljs-string">&quot;url&quot;</span>: <span class="hljs-string">&quot;ghcr.io/huggingface/text-generation-inference:sha-1734540&quot;</span>,  <span class="hljs-comment"># must be &gt;= 1.4.0</span>
<span class="hljs-meta">... </span>    },
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>endpoint.wait()
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(endpoint.status)`,wrap:!1}}),$=new d({props:{title:"2. Query the Inference Endpoint with OpenAI Client Libraries",local:"2-query-the-inference-endpoint-with-openai-client-libraries",headingTag:"h2"}}),H=new d({props:{title:"With the Python client",local:"with-the-python-client",headingTag:"h3"}}),L=new u({props:{code:"ZnJvbSUyMG9wZW5haSUyMGltcG9ydCUyME9wZW5BSSUwQSUwQUJBU0VfVVJMJTIwJTNEJTIwZW5kcG9pbnQudXJsJTBBJTBBJTIzJTIwaW5pdCUyMHRoZSUyMGNsaWVudCUyMGJ1dCUyMHBvaW50JTIwaXQlMjB0byUyMFRHSSUwQWNsaWVudCUyMCUzRCUyME9wZW5BSSglMEElMjAlMjAlMjAlMjBiYXNlX3VybCUzRG9zLnBhdGguam9pbihCQVNFX1VSTCUyQyUyMCUyMnYxJTJGJTIyKSUyQyUwQSUyMCUyMCUyMCUyMGFwaV9rZXklM0RIRl9BUElfS0VZJTJDJTBBKSUwQWNoYXRfY29tcGxldGlvbiUyMCUzRCUyMGNsaWVudC5jaGF0LmNvbXBsZXRpb25zLmNyZWF0ZSglMEElMjAlMjAlMjAlMjBtb2RlbCUzRCUyMnRnaSUyMiUyQyUwQSUyMCUyMCUyMCUyMG1lc3NhZ2VzJTNEJTVCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTdCJTIycm9sZSUyMiUzQSUyMCUyMnN5c3RlbSUyMiUyQyUyMCUyMmNvbnRlbnQlMjIlM0ElMjAlMjJZb3UlMjBhcmUlMjBhJTIwaGVscGZ1bCUyMGFzc2lzdGFudC4lMjIlN0QlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlN0IlMjJyb2xlJTIyJTNBJTIwJTIydXNlciUyMiUyQyUyMCUyMmNvbnRlbnQlMjIlM0ElMjAlMjJXaHklMjBpcyUyMG9wZW4tc291cmNlJTIwc29mdHdhcmUlMjBpbXBvcnRhbnQlM0YlMjIlN0QlMkMlMEElMjAlMjAlMjAlMjAlNUQlMkMlMEElMjAlMjAlMjAlMjBzdHJlYW0lM0RUcnVlJTJDJTBBJTIwJTIwJTIwJTIwbWF4X3Rva2VucyUzRDUwMCUyQyUwQSklMEElMEElMjMlMjBpdGVyYXRlJTIwYW5kJTIwcHJpbnQlMjBzdHJlYW0lMEFmb3IlMjBtZXNzYWdlJTIwaW4lMjBjaGF0X2NvbXBsZXRpb24lM0ElMEElMjAlMjAlMjAlMjBwcmludChtZXNzYWdlLmNob2ljZXMlNUIwJTVELmRlbHRhLmNvbnRlbnQlMkMlMjBlbmQlM0QlMjIlMjIp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> openai <span class="hljs-keyword">import</span> OpenAI

<span class="hljs-meta">&gt;&gt;&gt; </span>BASE_URL = endpoint.url

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># init the client but point it to TGI</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>client = OpenAI(
<span class="hljs-meta">... </span>    base_url=os.path.join(BASE_URL, <span class="hljs-string">&quot;v1/&quot;</span>),
<span class="hljs-meta">... </span>    api_key=HF_API_KEY,
<span class="hljs-meta">... </span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>chat_completion = client.chat.completions.create(
<span class="hljs-meta">... </span>    model=<span class="hljs-string">&quot;tgi&quot;</span>,
<span class="hljs-meta">... </span>    messages=[
<span class="hljs-meta">... </span>        {<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;system&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: <span class="hljs-string">&quot;You are a helpful assistant.&quot;</span>},
<span class="hljs-meta">... </span>        {<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;user&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: <span class="hljs-string">&quot;Why is open-source software important?&quot;</span>},
<span class="hljs-meta">... </span>    ],
<span class="hljs-meta">... </span>    stream=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>    max_tokens=<span class="hljs-number">500</span>,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># iterate and print stream</span>
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">for</span> message <span class="hljs-keyword">in</span> chat_completion:
<span class="hljs-meta">... </span>    <span class="hljs-built_in">print</span>(message.choices[<span class="hljs-number">0</span>].delta.content, end=<span class="hljs-string">&quot;&quot;</span>)`,wrap:!1}}),D=new d({props:{title:"With the JavaScript client",local:"with-the-javascript-client",headingTag:"h3"}}),ee=new u({props:{code:"aW1wb3J0JTIwT3BlbkFJJTIwZnJvbSUyMCUyMm9wZW5haSUyMiUzQiUwQSUwQWNvbnN0JTIwb3BlbmFpJTIwJTNEJTIwbmV3JTIwT3BlbkFJKCU3QiUwQSUyMCUyMGJhc2VVUkwlM0ElMjAlMjIlM0NFTkRQT0lOVF9VUkwlM0UlMjIlMjAlMkIlMjAlMjIlMkZ2MSUyRiUyMiUyQyUyMCUyRiUyRiUyMHJlcGxhY2UlMjB3aXRoJTIweW91ciUyMGVuZHBvaW50JTIwdXJsJTBBJTIwJTIwYXBpS2V5JTNBJTIwJTIyJTNDSEZfQVBJX1RPS0VOJTNFJTIyJTJDJTIwJTJGJTJGJTIwcmVwbGFjZSUyMHdpdGglMjB5b3VyJTIwdG9rZW4lMEElN0QpJTNCJTBBJTBBYXN5bmMlMjBmdW5jdGlvbiUyMG1haW4oKSUyMCU3QiUwQSUyMCUyMGNvbnN0JTIwc3RyZWFtJTIwJTNEJTIwYXdhaXQlMjBvcGVuYWkuY2hhdC5jb21wbGV0aW9ucy5jcmVhdGUoJTdCJTBBJTIwJTIwJTIwJTIwbW9kZWwlM0ElMjAlMjJ0Z2klMjIlMkMlMEElMjAlMjAlMjAlMjBtZXNzYWdlcyUzQSUyMCU1QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCU3QiUyMHJvbGUlM0ElMjAlMjJzeXN0ZW0lMjIlMkMlMjBjb250ZW50JTNBJTIwJTIyWW91JTIwYXJlJTIwYSUyMGhlbHBmdWwlMjBhc3Npc3RhbnQuJTIyJTIwJTdEJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTdCJTIwcm9sZSUzQSUyMCUyMnVzZXIlMjIlMkMlMjBjb250ZW50JTNBJTIwJTIyV2h5JTIwaXMlMjBvcGVuLXNvdXJjZSUyMHNvZnR3YXJlJTIwaW1wb3J0YW50JTNGJTIyJTIwJTdEJTJDJTBBJTIwJTIwJTIwJTIwJTVEJTJDJTBBJTIwJTIwJTIwJTIwc3RyZWFtJTNBJTIwdHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMG1heF90b2tlbnMlM0ElMjA1MDAlMkMlMEElMjAlMjAlN0QpJTNCJTBBJTIwJTIwZm9yJTIwYXdhaXQlMjAoY29uc3QlMjBjaHVuayUyMG9mJTIwc3RyZWFtKSUyMCU3QiUwQSUyMCUyMCUyMCUyMHByb2Nlc3Muc3Rkb3V0LndyaXRlKGNodW5rLmNob2ljZXMlNUIwJTVEJTNGLmRlbHRhJTNGLmNvbnRlbnQlMjAlN0MlN0MlMjAlMjIlMjIpJTNCJTBBJTIwJTIwJTdEJTBBJTdEJTBBJTBBbWFpbigpJTNC",highlighted:`<span class="hljs-keyword">import</span> <span class="hljs-title class_">OpenAI</span> <span class="hljs-keyword">from</span> <span class="hljs-string">&quot;openai&quot;</span>;

<span class="hljs-keyword">const</span> openai = <span class="hljs-keyword">new</span> <span class="hljs-title class_">OpenAI</span>({
  <span class="hljs-attr">baseURL</span>: <span class="hljs-string">&quot;&lt;ENDPOINT_URL&gt;&quot;</span> + <span class="hljs-string">&quot;/v1/&quot;</span>, <span class="hljs-comment">// replace with your endpoint url</span>
  <span class="hljs-attr">apiKey</span>: <span class="hljs-string">&quot;&lt;HF_API_TOKEN&gt;&quot;</span>, <span class="hljs-comment">// replace with your token</span>
});

<span class="hljs-keyword">async</span> <span class="hljs-keyword">function</span> <span class="hljs-title function_">main</span>(<span class="hljs-params"></span>) {
  <span class="hljs-keyword">const</span> stream = <span class="hljs-keyword">await</span> openai.<span class="hljs-property">chat</span>.<span class="hljs-property">completions</span>.<span class="hljs-title function_">create</span>({
    <span class="hljs-attr">model</span>: <span class="hljs-string">&quot;tgi&quot;</span>,
    <span class="hljs-attr">messages</span>: [
      { <span class="hljs-attr">role</span>: <span class="hljs-string">&quot;system&quot;</span>, <span class="hljs-attr">content</span>: <span class="hljs-string">&quot;You are a helpful assistant.&quot;</span> },
      { <span class="hljs-attr">role</span>: <span class="hljs-string">&quot;user&quot;</span>, <span class="hljs-attr">content</span>: <span class="hljs-string">&quot;Why is open-source software important?&quot;</span> },
    ],
    <span class="hljs-attr">stream</span>: <span class="hljs-literal">true</span>,
    <span class="hljs-attr">max_tokens</span>: <span class="hljs-number">500</span>,
  });
  <span class="hljs-keyword">for</span> <span class="hljs-keyword">await</span> (<span class="hljs-keyword">const</span> chunk <span class="hljs-keyword">of</span> stream) {
    process.<span class="hljs-property">stdout</span>.<span class="hljs-title function_">write</span>(chunk.<span class="hljs-property">choices</span>[<span class="hljs-number">0</span>]?.<span class="hljs-property">delta</span>?.<span class="hljs-property">content</span> || <span class="hljs-string">&quot;&quot;</span>);
  }
}

<span class="hljs-title function_">main</span>();`,wrap:!1}}),te=new d({props:{title:"3. Integrate with LangChain and LlamaIndex",local:"3-integrate-with-langchain-and-llamaindex",headingTag:"h2"}}),le=new d({props:{title:"How to use with LangChain",local:"how-to-use-with-langchain",headingTag:"h3"}}),ae=new u({props:{code:"ZnJvbSUyMGxhbmdjaGFpbl9vcGVuYWklMjBpbXBvcnQlMjBDaGF0T3BlbkFJJTBBJTBBbGxtJTIwJTNEJTIwQ2hhdE9wZW5BSSglMEElMjAlMjAlMjAlMjBtb2RlbF9uYW1lJTNEJTIydGdpJTIyJTJDJTBBJTIwJTIwJTIwJTIwb3BlbmFpX2FwaV9rZXklM0RIRl9BUElfS0VZJTJDJTBBJTIwJTIwJTIwJTIwb3BlbmFpX2FwaV9iYXNlJTNEb3MucGF0aC5qb2luKEJBU0VfVVJMJTJDJTIwJTIydjElMkYlMjIpJTJDJTBBKSUwQWxsbS5pbnZva2UoJTIyV2h5JTIwaXMlMjBvcGVuLXNvdXJjZSUyMHNvZnR3YXJlJTIwaW1wb3J0YW50JTNGJTIyKQ==",highlighted:`<span class="hljs-keyword">from</span> langchain_openai <span class="hljs-keyword">import</span> ChatOpenAI

llm = ChatOpenAI(
    model_name=<span class="hljs-string">&quot;tgi&quot;</span>,
    openai_api_key=HF_API_KEY,
    openai_api_base=os.path.join(BASE_URL, <span class="hljs-string">&quot;v1/&quot;</span>),
)
llm.invoke(<span class="hljs-string">&quot;Why is open-source software important?&quot;</span>)`,wrap:!1}}),pe=new u({props:{code:"ZnJvbSUyMGxhbmdjaGFpbiUyMGltcG9ydCUyMGh1YiUwQWZyb20lMjBsYW5nY2hhaW4udGV4dF9zcGxpdHRlciUyMGltcG9ydCUyMFJlY3Vyc2l2ZUNoYXJhY3RlclRleHRTcGxpdHRlciUwQWZyb20lMjBsYW5nY2hhaW5fY29tbXVuaXR5LmRvY3VtZW50X2xvYWRlcnMlMjBpbXBvcnQlMjBXZWJCYXNlTG9hZGVyJTBBZnJvbSUyMGxhbmdjaGFpbl9jb21tdW5pdHkudmVjdG9yc3RvcmVzJTIwaW1wb3J0JTIwQ2hyb21hJTBBZnJvbSUyMGxhbmdjaGFpbl9jb3JlLm91dHB1dF9wYXJzZXJzJTIwaW1wb3J0JTIwU3RyT3V0cHV0UGFyc2VyJTBBZnJvbSUyMGxhbmdjaGFpbl9jb3JlLnJ1bm5hYmxlcyUyMGltcG9ydCUyMFJ1bm5hYmxlUGFzc3Rocm91Z2glMEFmcm9tJTIwbGFuZ2NoYWluX2NvcmUucnVubmFibGVzJTIwaW1wb3J0JTIwUnVubmFibGVQYXJhbGxlbCUwQWZyb20lMjBsYW5nY2hhaW5fY29tbXVuaXR5LmVtYmVkZGluZ3MlMjBpbXBvcnQlMjBIdWdnaW5nRmFjZUVtYmVkZGluZ3MlMEElMEElMjMlMjBMb2FkJTJDJTIwY2h1bmslMjBhbmQlMjBpbmRleCUyMHRoZSUyMGNvbnRlbnRzJTIwb2YlMjB0aGUlMjBibG9nJTBBbG9hZGVyJTIwJTNEJTIwV2ViQmFzZUxvYWRlciglMEElMjAlMjAlMjAlMjB3ZWJfcGF0aHMlM0QoJTIyaHR0cHMlM0ElMkYlMkZodWdnaW5nZmFjZS5jbyUyRmJsb2clMkZvcGVuLXNvdXJjZS1sbG1zLWFzLWFnZW50cyUyMiUyQyklMkMlMEEpJTBBZG9jcyUyMCUzRCUyMGxvYWRlci5sb2FkKCklMEElMEElMjMlMjBkZWNsYXJlJTIwYW4lMjBIRiUyMGVtYmVkZGluZyUyMG1vZGVsJTBBaGZfZW1iZWRkaW5ncyUyMCUzRCUyMEh1Z2dpbmdGYWNlRW1iZWRkaW5ncyhtb2RlbF9uYW1lJTNEJTIyQkFBSSUyRmJnZS1sYXJnZS1lbi12MS41JTIyKSUwQSUwQXRleHRfc3BsaXR0ZXIlMjAlM0QlMjBSZWN1cnNpdmVDaGFyYWN0ZXJUZXh0U3BsaXR0ZXIoY2h1bmtfc2l6ZSUzRDUxMiUyQyUyMGNodW5rX292ZXJsYXAlM0QyMDApJTBBc3BsaXRzJTIwJTNEJTIwdGV4dF9zcGxpdHRlci5zcGxpdF9kb2N1bWVudHMoZG9jcyklMEF2ZWN0b3JzdG9yZSUyMCUzRCUyMENocm9tYS5mcm9tX2RvY3VtZW50cyhkb2N1bWVudHMlM0RzcGxpdHMlMkMlMjBlbWJlZGRpbmclM0RoZl9lbWJlZGRpbmdzKSUwQSUwQSUyMyUyMFJldHJpZXZlJTIwYW5kJTIwZ2VuZXJhdGUlMjB1c2luZyUyMHRoZSUyMHJlbGV2YW50JTIwc25pcHBldHMlMjBvZiUyMHRoZSUyMGJsb2clMEFyZXRyaWV2ZXIlMjAlM0QlMjB2ZWN0b3JzdG9yZS5hc19yZXRyaWV2ZXIoKSUwQXByb21wdCUyMCUzRCUyMGh1Yi5wdWxsKCUyMnJsbSUyRnJhZy1wcm9tcHQlMjIpJTBBJTBBJTBBZGVmJTIwZm9ybWF0X2RvY3MoZG9jcyklM0ElMEElMjAlMjAlMjAlMjByZXR1cm4lMjAlMjIlNUNuJTVDbiUyMi5qb2luKGRvYy5wYWdlX2NvbnRlbnQlMjBmb3IlMjBkb2MlMjBpbiUyMGRvY3MpJTBBJTBBJTBBcmFnX2NoYWluX2Zyb21fZG9jcyUyMCUzRCUyMCglMEElMjAlMjAlMjAlMjBSdW5uYWJsZVBhc3N0aHJvdWdoLmFzc2lnbihjb250ZXh0JTNEKGxhbWJkYSUyMHglM0ElMjBmb3JtYXRfZG9jcyh4JTVCJTIyY29udGV4dCUyMiU1RCkpKSUyMCU3QyUyMHByb21wdCUyMCU3QyUyMGxsbSUyMCU3QyUyMFN0ck91dHB1dFBhcnNlcigpJTBBKSUwQSUwQXJhZ19jaGFpbl93aXRoX3NvdXJjZSUyMCUzRCUyMFJ1bm5hYmxlUGFyYWxsZWwoJTdCJTIyY29udGV4dCUyMiUzQSUyMHJldHJpZXZlciUyQyUyMCUyMnF1ZXN0aW9uJTIyJTNBJTIwUnVubmFibGVQYXNzdGhyb3VnaCgpJTdEKS5hc3NpZ24oJTBBJTIwJTIwJTIwJTIwYW5zd2VyJTNEcmFnX2NoYWluX2Zyb21fZG9jcyUwQSklMEElMEFyYWdfY2hhaW5fd2l0aF9zb3VyY2UuaW52b2tlKCUyMkFjY29yZGluZyUyMHRvJTIwdGhpcyUyMGFydGljbGUlMjB3aGljaCUyMG9wZW4tc291cmNlJTIwbW9kZWwlMjBpcyUyMHRoZSUyMGJlc3QlMjBmb3IlMjBhbiUyMGFnZW50JTIwYmVoYXZpb3VyJTNGJTIyKQ==",highlighted:`<span class="hljs-keyword">from</span> langchain <span class="hljs-keyword">import</span> hub
<span class="hljs-keyword">from</span> langchain.text_splitter <span class="hljs-keyword">import</span> RecursiveCharacterTextSplitter
<span class="hljs-keyword">from</span> langchain_community.document_loaders <span class="hljs-keyword">import</span> WebBaseLoader
<span class="hljs-keyword">from</span> langchain_community.vectorstores <span class="hljs-keyword">import</span> Chroma
<span class="hljs-keyword">from</span> langchain_core.output_parsers <span class="hljs-keyword">import</span> StrOutputParser
<span class="hljs-keyword">from</span> langchain_core.runnables <span class="hljs-keyword">import</span> RunnablePassthrough
<span class="hljs-keyword">from</span> langchain_core.runnables <span class="hljs-keyword">import</span> RunnableParallel
<span class="hljs-keyword">from</span> langchain_community.embeddings <span class="hljs-keyword">import</span> HuggingFaceEmbeddings

<span class="hljs-comment"># Load, chunk and index the contents of the blog</span>
loader = WebBaseLoader(
    web_paths=(<span class="hljs-string">&quot;https://huggingface.co/blog/open-source-llms-as-agents&quot;</span>,),
)
docs = loader.load()

<span class="hljs-comment"># declare an HF embedding model</span>
hf_embeddings = HuggingFaceEmbeddings(model_name=<span class="hljs-string">&quot;BAAI/bge-large-en-v1.5&quot;</span>)

text_splitter = RecursiveCharacterTextSplitter(chunk_size=<span class="hljs-number">512</span>, chunk_overlap=<span class="hljs-number">200</span>)
splits = text_splitter.split_documents(docs)
vectorstore = Chroma.from_documents(documents=splits, embedding=hf_embeddings)

<span class="hljs-comment"># Retrieve and generate using the relevant snippets of the blog</span>
retriever = vectorstore.as_retriever()
prompt = hub.pull(<span class="hljs-string">&quot;rlm/rag-prompt&quot;</span>)


<span class="hljs-keyword">def</span> <span class="hljs-title function_">format_docs</span>(<span class="hljs-params">docs</span>):
    <span class="hljs-keyword">return</span> <span class="hljs-string">&quot;\\n\\n&quot;</span>.join(doc.page_content <span class="hljs-keyword">for</span> doc <span class="hljs-keyword">in</span> docs)


rag_chain_from_docs = (
    RunnablePassthrough.assign(context=(<span class="hljs-keyword">lambda</span> x: format_docs(x[<span class="hljs-string">&quot;context&quot;</span>]))) | prompt | llm | StrOutputParser()
)

rag_chain_with_source = RunnableParallel({<span class="hljs-string">&quot;context&quot;</span>: retriever, <span class="hljs-string">&quot;question&quot;</span>: RunnablePassthrough()}).assign(
    answer=rag_chain_from_docs
)

rag_chain_with_source.invoke(<span class="hljs-string">&quot;According to this article which open-source model is the best for an agent behaviour?&quot;</span>)`,wrap:!1}}),re=new d({props:{title:"How to use with LlamaIndex",local:"how-to-use-with-llamaindex",headingTag:"h3"}}),he=new u({props:{code:"ZnJvbSUyMGxsYW1hX2luZGV4LmxsbXMlMjBpbXBvcnQlMjBPcGVuQUlMaWtlJTBBJTBBbGxtJTIwJTNEJTIwT3BlbkFJTGlrZSglMEElMjAlMjAlMjAlMjBtb2RlbCUzRCUyMnRnaSUyMiUyQyUwQSUyMCUyMCUyMCUyMGFwaV9rZXklM0RIRl9BUElfS0VZJTJDJTBBJTIwJTIwJTIwJTIwYXBpX2Jhc2UlM0RCQVNFX1VSTCUyMCUyQiUyMCUyMiUyRnYxJTJGJTIyJTJDJTBBJTIwJTIwJTIwJTIwaXNfY2hhdF9tb2RlbCUzRFRydWUlMkMlMEElMjAlMjAlMjAlMjBpc19sb2NhbCUzREZhbHNlJTJDJTBBJTIwJTIwJTIwJTIwaXNfZnVuY3Rpb25fY2FsbGluZ19tb2RlbCUzREZhbHNlJTJDJTBBJTIwJTIwJTIwJTIwY29udGV4dF93aW5kb3clM0Q0MDk2JTJDJTBBKSUwQSUwQWxsbS5jb21wbGV0ZSglMjJXaHklMjBpcyUyMG9wZW4tc291cmNlJTIwc29mdHdhcmUlMjBpbXBvcnRhbnQlM0YlMjIp",highlighted:`<span class="hljs-keyword">from</span> llama_index.llms <span class="hljs-keyword">import</span> OpenAILike

llm = OpenAILike(
    model=<span class="hljs-string">&quot;tgi&quot;</span>,
    api_key=HF_API_KEY,
    api_base=BASE_URL + <span class="hljs-string">&quot;/v1/&quot;</span>,
    is_chat_model=<span class="hljs-literal">True</span>,
    is_local=<span class="hljs-literal">False</span>,
    is_function_calling_model=<span class="hljs-literal">False</span>,
    context_window=<span class="hljs-number">4096</span>,
)

llm.complete(<span class="hljs-string">&quot;Why is open-source software important?&quot;</span>)`,wrap:!1}}),me=new u({props:{code:"ZnJvbSUyMGxsYW1hX2luZGV4JTIwaW1wb3J0JTIwKCUwQSUyMCUyMCUyMCUyMFNlcnZpY2VDb250ZXh0JTJDJTBBJTIwJTIwJTIwJTIwVmVjdG9yU3RvcmVJbmRleCUyQyUwQSklMEFmcm9tJTIwbGxhbWFfaW5kZXglMjBpbXBvcnQlMjBkb3dubG9hZF9sb2FkZXIlMEFmcm9tJTIwbGxhbWFfaW5kZXguZW1iZWRkaW5ncyUyMGltcG9ydCUyMEh1Z2dpbmdGYWNlRW1iZWRkaW5nJTBBZnJvbSUyMGxsYW1hX2luZGV4LnF1ZXJ5X2VuZ2luZSUyMGltcG9ydCUyMENpdGF0aW9uUXVlcnlFbmdpbmUlMEElMEElMEFTaW1wbGVXZWJQYWdlUmVhZGVyJTIwJTNEJTIwZG93bmxvYWRfbG9hZGVyKCUyMlNpbXBsZVdlYlBhZ2VSZWFkZXIlMjIpJTBBJTBBZG9jdW1lbnRzJTIwJTNEJTIwU2ltcGxlV2ViUGFnZVJlYWRlcihodG1sX3RvX3RleHQlM0RUcnVlKS5sb2FkX2RhdGEoJTBBJTIwJTIwJTIwJTIwJTVCJTIyaHR0cHMlM0ElMkYlMkZodWdnaW5nZmFjZS5jbyUyRmJsb2clMkZvcGVuLXNvdXJjZS1sbG1zLWFzLWFnZW50cyUyMiU1RCUwQSklMEElMEElMjMlMjBMb2FkJTIwZW1iZWRkaW5nJTIwbW9kZWwlMEFlbWJlZF9tb2RlbCUyMCUzRCUyMEh1Z2dpbmdGYWNlRW1iZWRkaW5nKG1vZGVsX25hbWUlM0QlMjJCQUFJJTJGYmdlLWxhcmdlLWVuLXYxLjUlMjIpJTBBJTBBJTIzJTIwUGFzcyUyMExMTSUyMHRvJTIwcGlwZWxpbmUlMEFzZXJ2aWNlX2NvbnRleHQlMjAlM0QlMjBTZXJ2aWNlQ29udGV4dC5mcm9tX2RlZmF1bHRzKGVtYmVkX21vZGVsJTNEZW1iZWRfbW9kZWwlMkMlMjBsbG0lM0RsbG0pJTBBaW5kZXglMjAlM0QlMjBWZWN0b3JTdG9yZUluZGV4LmZyb21fZG9jdW1lbnRzKGRvY3VtZW50cyUyQyUyMHNlcnZpY2VfY29udGV4dCUzRHNlcnZpY2VfY29udGV4dCUyQyUyMHNob3dfcHJvZ3Jlc3MlM0RUcnVlKSUwQSUwQSUyMyUyMFF1ZXJ5JTIwdGhlJTIwaW5kZXglMEFxdWVyeV9lbmdpbmUlMjAlM0QlMjBDaXRhdGlvblF1ZXJ5RW5naW5lLmZyb21fYXJncyglMEElMjAlMjAlMjAlMjBpbmRleCUyQyUwQSUyMCUyMCUyMCUyMHNpbWlsYXJpdHlfdG9wX2slM0QyJTJDJTBBKSUwQXJlc3BvbnNlJTIwJTNEJTIwcXVlcnlfZW5naW5lLnF1ZXJ5KCUyMkFjY29yZGluZyUyMHRvJTIwdGhpcyUyMGFydGljbGUlMjB3aGljaCUyMG9wZW4tc291cmNlJTIwbW9kZWwlMjBpcyUyMHRoZSUyMGJlc3QlMjBmb3IlMjBhbiUyMGFnZW50JTIwYmVoYXZpb3VyJTNGJTIyKSUwQSUwQXJlc3BvbnNlLnJlc3BvbnNl",highlighted:`<span class="hljs-keyword">from</span> llama_index <span class="hljs-keyword">import</span> (
    ServiceContext,
    VectorStoreIndex,
)
<span class="hljs-keyword">from</span> llama_index <span class="hljs-keyword">import</span> download_loader
<span class="hljs-keyword">from</span> llama_index.embeddings <span class="hljs-keyword">import</span> HuggingFaceEmbedding
<span class="hljs-keyword">from</span> llama_index.query_engine <span class="hljs-keyword">import</span> CitationQueryEngine


SimpleWebPageReader = download_loader(<span class="hljs-string">&quot;SimpleWebPageReader&quot;</span>)

documents = SimpleWebPageReader(html_to_text=<span class="hljs-literal">True</span>).load_data(
    [<span class="hljs-string">&quot;https://huggingface.co/blog/open-source-llms-as-agents&quot;</span>]
)

<span class="hljs-comment"># Load embedding model</span>
embed_model = HuggingFaceEmbedding(model_name=<span class="hljs-string">&quot;BAAI/bge-large-en-v1.5&quot;</span>)

<span class="hljs-comment"># Pass LLM to pipeline</span>
service_context = ServiceContext.from_defaults(embed_model=embed_model, llm=llm)
index = VectorStoreIndex.from_documents(documents, service_context=service_context, show_progress=<span class="hljs-literal">True</span>)

<span class="hljs-comment"># Query the index</span>
query_engine = CitationQueryEngine.from_args(
    index,
    similarity_top_k=<span class="hljs-number">2</span>,
)
response = query_engine.query(<span class="hljs-string">&quot;According to this article which open-source model is the best for an agent behaviour?&quot;</span>)

response.response`,wrap:!1}}),ue=new d({props:{title:"Wrap up",local:"wrap-up",headingTag:"h2"}}),we=new u({props:{code:"JTIzJTIwcGF1c2UlMjBvdXIlMjBydW5uaW5nJTIwZW5kcG9pbnQlMEFlbmRwb2ludC5wYXVzZSgpJTBBJTBBJTIzJTIwb3B0aW9uYWxseSUyMGRlbGV0ZSUwQSUyMyUyMGVuZHBvaW50LmRlbGV0ZSgp",highlighted:`<span class="hljs-comment"># pause our running endpoint</span>
endpoint.pause()

<span class="hljs-comment"># optionally delete</span>
<span class="hljs-comment"># endpoint.delete()</span>`,wrap:!1}}),{c(){w=i("meta"),Ie=n(),Ue=i("p"),Be=n(),r(J.$$.fragment),Ce=n(),r(j.$$.fragment),Ze=n(),T=i("p"),T.innerHTML=kt,Ge=n(),U=i("p"),U.textContent=_t,We=n(),b=i("p"),b.innerHTML=Rt,ve=n(),g=i("p"),g.textContent=Et,ke=n(),f=i("ul"),f.innerHTML=At,_e=n(),I=i("p"),I.textContent=Nt,Re=n(),B=i("ol"),B.innerHTML=xt,Ee=n(),C=i("p"),C.innerHTML=Vt,Ae=n(),r(Z.$$.fragment),Ne=n(),G=i("p"),G.textContent=Xt,xe=n(),r(W.$$.fragment),Ve=n(),r(v.$$.fragment),Xe=n(),Je=i("a"),Qe=n(),r(k.$$.fragment),Se=n(),_=i("p"),_.innerHTML=Qt,$e=n(),R=i("p"),R.innerHTML=St,Fe=n(),E=i("p"),E.innerHTML=$t,He=n(),A=i("p"),A.innerHTML=Ft,qe=n(),r(N.$$.fragment),Ye=n(),x=i("pre"),x.textContent=Ht,Le=n(),V=i("p"),V.innerHTML=qt,ze=n(),X=i("p"),X.innerHTML=Yt,Pe=n(),Q=i("p"),Q.textContent=Lt,Oe=n(),S=i("p"),S.innerHTML=zt,De=n(),je=i("a"),Ke=n(),r($.$$.fragment),et=n(),F=i("p"),F.textContent=Pt,tt=n(),r(H.$$.fragment),st=n(),q=i("p"),q.innerHTML=Ot,lt=n(),Y=i("p"),Y.textContent=Dt,nt=n(),r(L.$$.fragment),at=n(),z=i("pre"),z.textContent=Kt,it=n(),P=i("p"),P.innerHTML=es,ot=n(),O=i("p"),O.innerHTML=ts,pt=n(),r(D.$$.fragment),rt=n(),K=i("p"),K.innerHTML=ss,ct=n(),r(ee.$$.fragment),Mt=n(),Te=i("a"),ht=n(),r(te.$$.fragment),yt=n(),se=i("p"),se.textContent=ls,mt=n(),r(le.$$.fragment),ut=n(),ne=i("p"),ne.innerHTML=ns,dt=n(),r(ae.$$.fragment),wt=n(),ie=i("p"),ie.innerHTML=as,Jt=n(),oe=i("p"),oe.textContent=is,jt=n(),r(pe.$$.fragment),Tt=n(),r(re.$$.fragment),Ut=n(),ce=i("p"),ce.innerHTML=os,bt=n(),Me=i("p"),Me.innerHTML=ps,gt=n(),r(he.$$.fragment),ft=n(),ye=i("p"),ye.innerHTML=rs,It=n(),r(me.$$.fragment),Bt=n(),r(ue.$$.fragment),Ct=n(),de=i("p"),de.textContent=cs,Zt=n(),r(we.$$.fragment),Gt=n(),be=i("p"),this.h()},l(e){const t=ds("svelte-u9bgzb",document.head);w=o(t,"META",{name:!0,content:!0}),t.forEach(s),Ie=a(e),Ue=o(e,"P",{}),ge(Ue).forEach(s),Be=a(e),c(J.$$.fragment,e),Ce=a(e),c(j.$$.fragment,e),Ze=a(e),T=o(e,"P",{"data-svelte-h":!0}),p(T)!=="svelte-15bmh6z"&&(T.innerHTML=kt),Ge=a(e),U=o(e,"P",{"data-svelte-h":!0}),p(U)!=="svelte-ifj9j3"&&(U.textContent=_t),We=a(e),b=o(e,"P",{"data-svelte-h":!0}),p(b)!=="svelte-19stfnn"&&(b.innerHTML=Rt),ve=a(e),g=o(e,"P",{"data-svelte-h":!0}),p(g)!=="svelte-1h5h72d"&&(g.textContent=Et),ke=a(e),f=o(e,"UL",{"data-svelte-h":!0}),p(f)!=="svelte-o7yv73"&&(f.innerHTML=At),_e=a(e),I=o(e,"P",{"data-svelte-h":!0}),p(I)!=="svelte-i15w4q"&&(I.textContent=Nt),Re=a(e),B=o(e,"OL",{"data-svelte-h":!0}),p(B)!=="svelte-dmuq8i"&&(B.innerHTML=xt),Ee=a(e),C=o(e,"P",{"data-svelte-h":!0}),p(C)!=="svelte-1yaed00"&&(C.innerHTML=Vt),Ae=a(e),c(Z.$$.fragment,e),Ne=a(e),G=o(e,"P",{"data-svelte-h":!0}),p(G)!=="svelte-1mbafnu"&&(G.textContent=Xt),xe=a(e),c(W.$$.fragment,e),Ve=a(e),c(v.$$.fragment,e),Xe=a(e),Je=o(e,"A",{id:!0}),ge(Je).forEach(s),Qe=a(e),c(k.$$.fragment,e),Se=a(e),_=o(e,"P",{"data-svelte-h":!0}),p(_)!=="svelte-1srz11y"&&(_.innerHTML=Qt),$e=a(e),R=o(e,"P",{"data-svelte-h":!0}),p(R)!=="svelte-1nafhc7"&&(R.innerHTML=St),Fe=a(e),E=o(e,"P",{"data-svelte-h":!0}),p(E)!=="svelte-1qq4fhb"&&(E.innerHTML=$t),He=a(e),A=o(e,"P",{"data-svelte-h":!0}),p(A)!=="svelte-1vrlf2e"&&(A.innerHTML=Ft),qe=a(e),c(N.$$.fragment,e),Ye=a(e),x=o(e,"PRE",{"data-svelte-h":!0}),p(x)!=="svelte-105t8kl"&&(x.textContent=Ht),Le=a(e),V=o(e,"P",{"data-svelte-h":!0}),p(V)!=="svelte-13u8bul"&&(V.innerHTML=qt),ze=a(e),X=o(e,"P",{"data-svelte-h":!0}),p(X)!=="svelte-8mpmx4"&&(X.innerHTML=Yt),Pe=a(e),Q=o(e,"P",{"data-svelte-h":!0}),p(Q)!=="svelte-1hh5tjb"&&(Q.textContent=Lt),Oe=a(e),S=o(e,"P",{"data-svelte-h":!0}),p(S)!=="svelte-1nk8k00"&&(S.innerHTML=zt),De=a(e),je=o(e,"A",{id:!0}),ge(je).forEach(s),Ke=a(e),c($.$$.fragment,e),et=a(e),F=o(e,"P",{"data-svelte-h":!0}),p(F)!=="svelte-m2ihxo"&&(F.textContent=Pt),tt=a(e),c(H.$$.fragment,e),st=a(e),q=o(e,"P",{"data-svelte-h":!0}),p(q)!=="svelte-1vvrp5u"&&(q.innerHTML=Ot),lt=a(e),Y=o(e,"P",{"data-svelte-h":!0}),p(Y)!=="svelte-5sa4me"&&(Y.textContent=Dt),nt=a(e),c(L.$$.fragment,e),at=a(e),z=o(e,"PRE",{"data-svelte-h":!0}),p(z)!=="svelte-1gevisn"&&(z.textContent=Kt),it=a(e),P=o(e,"P",{"data-svelte-h":!0}),p(P)!=="svelte-9ab64r"&&(P.innerHTML=es),ot=a(e),O=o(e,"P",{"data-svelte-h":!0}),p(O)!=="svelte-15i27pe"&&(O.innerHTML=ts),pt=a(e),c(D.$$.fragment,e),rt=a(e),K=o(e,"P",{"data-svelte-h":!0}),p(K)!=="svelte-a5y7nz"&&(K.innerHTML=ss),ct=a(e),c(ee.$$.fragment,e),Mt=a(e),Te=o(e,"A",{id:!0}),ge(Te).forEach(s),ht=a(e),c(te.$$.fragment,e),yt=a(e),se=o(e,"P",{"data-svelte-h":!0}),p(se)!=="svelte-1i9oels"&&(se.textContent=ls),mt=a(e),c(le.$$.fragment,e),ut=a(e),ne=o(e,"P",{"data-svelte-h":!0}),p(ne)!=="svelte-1qdzd81"&&(ne.innerHTML=ns),dt=a(e),c(ae.$$.fragment,e),wt=a(e),ie=o(e,"P",{"data-svelte-h":!0}),p(ie)!=="svelte-4flxc0"&&(ie.innerHTML=as),Jt=a(e),oe=o(e,"P",{"data-svelte-h":!0}),p(oe)!=="svelte-c9m7s5"&&(oe.textContent=is),jt=a(e),c(pe.$$.fragment,e),Tt=a(e),c(re.$$.fragment,e),Ut=a(e),ce=o(e,"P",{"data-svelte-h":!0}),p(ce)!=="svelte-gt52es"&&(ce.innerHTML=os),bt=a(e),Me=o(e,"P",{"data-svelte-h":!0}),p(Me)!=="svelte-1oxfz4m"&&(Me.innerHTML=ps),gt=a(e),c(he.$$.fragment,e),ft=a(e),ye=o(e,"P",{"data-svelte-h":!0}),p(ye)!=="svelte-1yqegdk"&&(ye.innerHTML=rs),It=a(e),c(me.$$.fragment,e),Bt=a(e),c(ue.$$.fragment,e),Ct=a(e),de=o(e,"P",{"data-svelte-h":!0}),p(de)!=="svelte-132npwv"&&(de.textContent=cs),Zt=a(e),c(we.$$.fragment,e),Gt=a(e),be=o(e,"P",{}),ge(be).forEach(s),this.h()},h(){fe(w,"name","hf:doc:metadata"),fe(w,"content",Ts),fe(Je,"id","section_1"),fe(je,"id","section_2"),fe(Te,"id","section_3")},m(e,t){ws(document.head,w),l(e,Ie,t),l(e,Ue,t),l(e,Be,t),M(J,e,t),l(e,Ce,t),M(j,e,t),l(e,Ze,t),l(e,T,t),l(e,Ge,t),l(e,U,t),l(e,We,t),l(e,b,t),l(e,ve,t),l(e,g,t),l(e,ke,t),l(e,f,t),l(e,_e,t),l(e,I,t),l(e,Re,t),l(e,B,t),l(e,Ee,t),l(e,C,t),l(e,Ae,t),M(Z,e,t),l(e,Ne,t),l(e,G,t),l(e,xe,t),M(W,e,t),l(e,Ve,t),M(v,e,t),l(e,Xe,t),l(e,Je,t),l(e,Qe,t),M(k,e,t),l(e,Se,t),l(e,_,t),l(e,$e,t),l(e,R,t),l(e,Fe,t),l(e,E,t),l(e,He,t),l(e,A,t),l(e,qe,t),M(N,e,t),l(e,Ye,t),l(e,x,t),l(e,Le,t),l(e,V,t),l(e,ze,t),l(e,X,t),l(e,Pe,t),l(e,Q,t),l(e,Oe,t),l(e,S,t),l(e,De,t),l(e,je,t),l(e,Ke,t),M($,e,t),l(e,et,t),l(e,F,t),l(e,tt,t),M(H,e,t),l(e,st,t),l(e,q,t),l(e,lt,t),l(e,Y,t),l(e,nt,t),M(L,e,t),l(e,at,t),l(e,z,t),l(e,it,t),l(e,P,t),l(e,ot,t),l(e,O,t),l(e,pt,t),M(D,e,t),l(e,rt,t),l(e,K,t),l(e,ct,t),M(ee,e,t),l(e,Mt,t),l(e,Te,t),l(e,ht,t),M(te,e,t),l(e,yt,t),l(e,se,t),l(e,mt,t),M(le,e,t),l(e,ut,t),l(e,ne,t),l(e,dt,t),M(ae,e,t),l(e,wt,t),l(e,ie,t),l(e,Jt,t),l(e,oe,t),l(e,jt,t),M(pe,e,t),l(e,Tt,t),M(re,e,t),l(e,Ut,t),l(e,ce,t),l(e,bt,t),l(e,Me,t),l(e,gt,t),M(he,e,t),l(e,ft,t),l(e,ye,t),l(e,It,t),M(me,e,t),l(e,Bt,t),M(ue,e,t),l(e,Ct,t),l(e,de,t),l(e,Zt,t),M(we,e,t),l(e,Gt,t),l(e,be,t),Wt=!0},p:hs,i(e){Wt||(h(J.$$.fragment,e),h(j.$$.fragment,e),h(Z.$$.fragment,e),h(W.$$.fragment,e),h(v.$$.fragment,e),h(k.$$.fragment,e),h(N.$$.fragment,e),h($.$$.fragment,e),h(H.$$.fragment,e),h(L.$$.fragment,e),h(D.$$.fragment,e),h(ee.$$.fragment,e),h(te.$$.fragment,e),h(le.$$.fragment,e),h(ae.$$.fragment,e),h(pe.$$.fragment,e),h(re.$$.fragment,e),h(he.$$.fragment,e),h(me.$$.fragment,e),h(ue.$$.fragment,e),h(we.$$.fragment,e),Wt=!0)},o(e){y(J.$$.fragment,e),y(j.$$.fragment,e),y(Z.$$.fragment,e),y(W.$$.fragment,e),y(v.$$.fragment,e),y(k.$$.fragment,e),y(N.$$.fragment,e),y($.$$.fragment,e),y(H.$$.fragment,e),y(L.$$.fragment,e),y(D.$$.fragment,e),y(ee.$$.fragment,e),y(te.$$.fragment,e),y(le.$$.fragment,e),y(ae.$$.fragment,e),y(pe.$$.fragment,e),y(re.$$.fragment,e),y(he.$$.fragment,e),y(me.$$.fragment,e),y(ue.$$.fragment,e),y(we.$$.fragment,e),Wt=!1},d(e){e&&(s(Ie),s(Ue),s(Be),s(Ce),s(Ze),s(T),s(Ge),s(U),s(We),s(b),s(ve),s(g),s(ke),s(f),s(_e),s(I),s(Re),s(B),s(Ee),s(C),s(Ae),s(Ne),s(G),s(xe),s(Ve),s(Xe),s(Je),s(Qe),s(Se),s(_),s($e),s(R),s(Fe),s(E),s(He),s(A),s(qe),s(Ye),s(x),s(Le),s(V),s(ze),s(X),s(Pe),s(Q),s(Oe),s(S),s(De),s(je),s(Ke),s(et),s(F),s(tt),s(st),s(q),s(lt),s(Y),s(nt),s(at),s(z),s(it),s(P),s(ot),s(O),s(pt),s(rt),s(K),s(ct),s(Mt),s(Te),s(ht),s(yt),s(se),s(mt),s(ut),s(ne),s(dt),s(wt),s(ie),s(Jt),s(oe),s(jt),s(Tt),s(Ut),s(ce),s(bt),s(Me),s(gt),s(ft),s(ye),s(It),s(Bt),s(Ct),s(de),s(Zt),s(Gt),s(be)),s(w),m(J,e),m(j,e),m(Z,e),m(W,e),m(v,e),m(k,e),m(N,e),m($,e),m(H,e),m(L,e),m(D,e),m(ee,e),m(te,e),m(le,e),m(ae,e),m(pe,e),m(re,e),m(he,e),m(me,e),m(ue,e),m(we,e)}}}const Ts='{"title":"Migrating from OpenAI to Open LLMs Using TGI’s Messages API","local":"migrating-from-openai-to-open-llms-using-tgis-messages-api","sections":[{"title":"Setup","local":"setup","sections":[],"depth":2},{"title":"1. Create an Inference Endpoint","local":"1-create-an-inference-endpoint","sections":[],"depth":2},{"title":"2. Query the Inference Endpoint with OpenAI Client Libraries","local":"2-query-the-inference-endpoint-with-openai-client-libraries","sections":[{"title":"With the Python client","local":"with-the-python-client","sections":[],"depth":3},{"title":"With the JavaScript client","local":"with-the-javascript-client","sections":[],"depth":3}],"depth":2},{"title":"3. Integrate with LangChain and LlamaIndex","local":"3-integrate-with-langchain-and-llamaindex","sections":[{"title":"How to use with LangChain","local":"how-to-use-with-langchain","sections":[],"depth":3},{"title":"How to use with LlamaIndex","local":"how-to-use-with-llamaindex","sections":[],"depth":3}],"depth":2},{"title":"Wrap up","local":"wrap-up","sections":[],"depth":2}],"depth":1}';function Us(vt){return ys(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Bs extends ms{constructor(w){super(),us(this,w,Us,js,Ms,{})}}export{Bs as component};
