Quick code tutorial: Stream text

To do stream generated text, add the following code to the example.py file. Replace {watsonx_ai_url}, {apikey}, and {project_id} with your values.

from ibm_watsonx_ai import Credentials
from ibm_watsonx_ai.foundation_models import ModelInference
import json

credentials = Credentials(
   api_key = "{apikey}", 
   url = "{watsonx_ai_url}" 
)

model_inference = ModelInference(
   model_id    = "ibm/granite-13b-chat-v2",
   params      = { "max_new_tokens" : 30 },
   credentials = credentials,
   project_id  = "{project_id}"
)

prompt = "Write short email announcing simply that the cafeteria will be closed today to be decorated for the staff party and be open tomorrow"

result = model.generate_text_stream( prompt = prompt )

for chunk in generated_response:
   print( chunk, end="\n", flush=True )

Sample output:

The
 cafe
t
eria
 will
 be
 closed
 today
 to
 be
 decorated
 for
 the
 staff
 party
.
 It
 will
 be
 open
 tomorrow
.
 Thanks
.

To do stream generated text, add the following code to the example.js file. Replace {apikey} and {project_id} with your values.

const { IamAuthenticator } = require( "ibm-cloud-sdk-core" );
const { WatsonXAI } = require( "@ibm-cloud/watsonx-ai" );

const authenticator = new IamAuthenticator( {
   "apikey" : "{apikey}"
} );

async function streamText()
{
    const text_stream = await watsonxAI_service.generateTextStream( {
       "input"     : "Write short email announcing simply that the cafeteria will be closed today to be decorated for the staff party and be open tomorrow",
       "modelId"   : "google/flan-t5-xxl",
       "parameters"   : { "max_new_tokens" : 30 },
       "returnObject" : true,
       "projectId"    : "{project_id}"
    } );
 
    for await ( const obj of text_stream )
    {
      console.log( obj.data.results[0].generated_text );
    }
}

streamText();

Sample output:

The
 cafe
t
eria
 will
 be
 closed
 today
 to
 be
 decorated
 for
 the
 staff
 party
.
 It
 will
 be
 open
 tomorrow
.
 Thanks
.

To stream generated text:

1. Create a JSON file with the following content. Replace {project_id} with your value.

   {
   "input" : "Write short email announcing simply that the cafeteria will be closed today to be decorated for the staff party and be open tomorrow",
   "model_id" : "google/flan-t5-xxl",
   "parameters": { "max_new_tokens": 30 },
   "project_id" : "<your-project-id>"
   }
   
   

2. Run the following curl request. Replace {token} and {JSON-file} with your values.

   curl -X POST                                                     \
   -H "Authorization: Bearer {token}"                          \
   -H "Content-Type: application/json"                              \
   <your-service-url>/ml/v1/text/generation_stream?version=2024-05-31 \
   --data "@{JSON-file}"
   
   

Sample output:

id: 1
event: message
data: { "model_id":"google/flan-t5-xxl",
        "created_at":"2025-03-01T02:50:13.751Z",
        "results":[ { "generated_text":"",
                      "generated_token_count":0,
                      "input_token_count":28,
                      "stop_reason":"not_finished" } ],
        "system":{...} }

id: 2
event: message
data: { "model_id":"google/flan-t5-xxl",
        "created_at":"2025-03-01T02:50:13.798Z",
        "results":[ { "generated_text":"The",
                      "generated_token_count":1,
                      "input_token_count":0,
                      "stop_reason":"not_finished" } ] }

id: 3
event: message
data: { "model_id":"google/flan-t5-xxl",
        "created_at":"2025-03-01T02:50:13.821Z",
        "results":[ { "generated_text":" cafe",
                      "generated_token_count":2,
                      "input_token_count":0,
                      "stop_reason":"not_finished" } ] }

id: 4
event: message
data: { "model_id":"google/flan-t5-xxl",
        "created_at":"2025-03-01T02:50:13.844Z",
        "results":[ { "generated_text":"t",
                      "generated_token_count":3,
                      "input_token_count":0,
                      "stop_reason":"not_finished" } ] }

id: 5
event: message
data: { "model_id":"google/flan-t5-xxl",
        "created_at":"2025-03-01T02:50:13.867Z",
        "results":[ { "generated_text":"eria",
                      "generated_token_count":4,
                      "input_token_count":0,
                      "stop_reason":"not_finished" } ] }

id: 6
event: message
data: { "model_id":"google/flan-t5-xxl",
        "created_at":"2025-03-01T02:50:13.889Z",
        "results":[ { "generated_text":" will",
                      "generated_token_count":5,
                      "input_token_count":0,
                      "stop_reason":"not_finished" } ] }

...